From bde00952b6aa68372e739e1f377cc9c32a9063f1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 23:19:21 +0000 Subject: [PATCH 001/672] fix(proxy): requeue Redis spend buffer transactions when DB commit fails The Redis transaction buffer leader drains the spend buffers with a destructive lpop before committing to the database. When the DB commit failed after exhausting retries, the popped transactions were only logged and then lost, permanently undercounting key/user/team/org/end-user/ team-member/tag/agent and daily spend after a database outage. Track each popped category and re-push the ones that were not committed back to their Redis buffers so a later scheduler tick retries them. Categories that already committed are not re-queued, so their spend is not double-counted. The daily tag spend path gets the same treatment. --- litellm/proxy/db/db_spend_update_writer.py | 45 +++++- .../redis_update_buffer.py | 55 +++++++ .../test_redis_update_buffer.py | 46 ++++++ .../proxy/db/test_db_spend_update_writer.py | 151 +++++++++++++++++- 4 files changed, 289 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 54a4c2dad91..cc266019ff7 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -797,6 +797,12 @@ class DBSpendUpdateWriter: ): verbose_proxy_logger.debug("acquired lock for spend updates") + # Track everything popped from Redis. Each category is removed once it + # has been committed to the DB, so whatever is left after a failure can + # be re-queued for the next tick instead of being lost. Committed + # categories are never re-queued, so their spend is not double-counted. + uncommitted: dict[str, Any] = {} # mutable-ok: drives which popped categories still need re-queuing + try: ( db_spend_update_transactions, @@ -807,6 +813,15 @@ class DBSpendUpdateWriter: daily_agent_spend_update_transactions, ) = await self.redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline() + uncommitted = { # mutable-ok: drives which popped categories still need re-queuing + "db_spend_update_transactions": db_spend_update_transactions, + "daily_spend_update_transactions": daily_spend_update_transactions, + "daily_team_spend_update_transactions": daily_team_spend_update_transactions, + "daily_org_spend_update_transactions": daily_org_spend_update_transactions, + "daily_end_user_spend_update_transactions": daily_end_user_spend_update_transactions, + "daily_agent_spend_update_transactions": daily_agent_spend_update_transactions, + } + if db_spend_update_transactions is not None: verbose_proxy_logger.info( "Spend tracking - committing spend updates from Redis to DB: " @@ -826,6 +841,7 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, db_spend_update_transactions=db_spend_update_transactions, ) + uncommitted.pop("db_spend_update_transactions", None) if daily_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_user_spend( @@ -834,6 +850,8 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_spend_update_transactions, ) + uncommitted.pop("daily_spend_update_transactions", None) + if daily_team_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_team_spend( n_retry_times=n_retry_times, @@ -841,6 +859,7 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_team_spend_update_transactions, ) + uncommitted.pop("daily_team_spend_update_transactions", None) if daily_org_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_org_spend( @@ -849,6 +868,7 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_org_spend_update_transactions, ) + uncommitted.pop("daily_org_spend_update_transactions", None) if daily_end_user_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_end_user_spend( @@ -857,6 +877,8 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_end_user_spend_update_transactions, ) + uncommitted.pop("daily_end_user_spend_update_transactions", None) + if daily_agent_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_agent_spend( n_retry_times=n_retry_times, @@ -864,14 +886,20 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_agent_spend_update_transactions, ) + uncommitted.pop("daily_agent_spend_update_transactions", None) except Exception as e: spend_log_error( "Spend tracking - failed to commit spend updates from Redis to DB. " - "Data already popped from Redis may be lost. Error: %s", + "Re-queuing uncommitted transactions to Redis for retry on next tick. Error: %s", str(e), exc=e, ) finally: + to_restore = { # mutable-ok: transient kwargs payload consumed immediately below + name: txns for name, txns in uncommitted.items() if txns is not None + } + if to_restore: + await self.redis_update_buffer.restore_transactions_to_redis(**to_restore) await self.pod_lock_manager.release_lock( cronjob_id=DB_SPEND_UPDATE_JOB_NAME, ) @@ -1020,11 +1048,11 @@ class DBSpendUpdateWriter: cronjob_id=DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, ): verbose_proxy_logger.debug("acquired lock for daily tag spend updates") + daily_tag_spend_update_transactions = ( + await self.redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + ) + committed = False try: - daily_tag_spend_update_transactions = ( - await self.redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() - ) - if daily_tag_spend_update_transactions: await DBSpendUpdateWriter.update_daily_tag_spend( n_retry_times=n_retry_times, @@ -1032,14 +1060,19 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_tag_spend_update_transactions, ) + committed = True except Exception as e: spend_log_error( "Spend tracking - failed to commit daily tag spend updates from Redis to DB. " - "Data already popped from Redis may be lost. Error: %s", + "Re-queuing to Redis for retry on next tick. Error: %s", str(e), exc=e, ) finally: + if not committed and daily_tag_spend_update_transactions: + await self.redis_update_buffer.restore_transactions_to_redis( + daily_tag_spend_update_transactions=daily_tag_spend_update_transactions, + ) await self.pod_lock_manager.release_lock( cronjob_id=DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, ) diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index c924448669d..b30fadd86ab 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -8,6 +8,8 @@ import asyncio import json from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from redis.exceptions import RedisError + from litellm._logging import verbose_proxy_logger from litellm.caching import RedisCache from litellm.constants import ( @@ -374,6 +376,59 @@ class RedisUpdateBuffer: if daily_txns: await daily_queue.update_queue.put(daily_txns) + async def restore_transactions_to_redis( + self, + db_spend_update_transactions: DBSpendUpdateTransactions | None = None, + daily_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + daily_team_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + daily_org_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + daily_end_user_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + daily_agent_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + daily_tag_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + ) -> None: + """ + Re-push transactions that were popped from Redis but not committed to the DB. + + The leader drains the buffers with a destructive ``lpop`` before committing to + the database. When a commit fails after its retries are exhausted, the popped + transactions must be pushed back so a later scheduler tick can retry them; + otherwise the aggregated spend is lost permanently. The re-pushed payloads use + the same JSON encoding as the store path, so the next drain parses them normally. + """ + if self.redis_cache is None: + return + + _configs = ( + (db_spend_update_transactions, REDIS_UPDATE_BUFFER_KEY), + (daily_spend_update_transactions, REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY), + (daily_team_spend_update_transactions, REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY), + (daily_org_spend_update_transactions, REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY), + (daily_end_user_spend_update_transactions, REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY), + (daily_agent_spend_update_transactions, REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY), + (daily_tag_spend_update_transactions, REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY), + ) + + rpush_list: list[RedisPipelineRpushOperation] = [ # mutable-ok: async_rpush_pipeline requires a list arg + RedisPipelineRpushOperation(key=redis_key, values=[safe_dumps(transactions)]) + for transactions, redis_key in _configs + if transactions + ] + if len(rpush_list) == 0: + return + + try: + await self.redis_cache.async_rpush_pipeline(rpush_list=rpush_list) + verbose_proxy_logger.info( + "Spend tracking - restored %d uncommitted transaction set(s) to Redis for retry on next tick.", + len(rpush_list), + ) + except RedisError as e: + verbose_proxy_logger.error( + "Spend tracking - failed to restore uncommitted transactions to Redis. " + "These spend updates are lost. Error: %s", + str(e), + ) + @staticmethod def _number_of_transactions_to_store_in_redis( db_spend_update_transactions: DBSpendUpdateTransactions, diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 33372e7794a..79909561683 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -270,6 +270,52 @@ async def test_get_all_transactions_from_redis_buffer_pipeline_no_redis(): assert result == (None, None, None, None, None, None) +@pytest.mark.asyncio +async def test_restore_transactions_to_redis_pushes_only_provided( + redis_update_buffer, mock_redis_cache +): + """ + restore_transactions_to_redis re-pushes only the transaction sets it was + given, to their matching buffer keys, so uncommitted spend can be retried. + """ + from litellm.constants import ( + REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY, + REDIS_UPDATE_BUFFER_KEY, + ) + + mock_redis_cache.async_rpush_pipeline = AsyncMock(return_value=[1, 1]) + + db_spend = {"key_list_transactions": {"key1": 1.0}} + daily_user = {"user_key1": {"spend": 1.0}} + + await redis_update_buffer.restore_transactions_to_redis( + db_spend_update_transactions=db_spend, + daily_spend_update_transactions=daily_user, + ) + + mock_redis_cache.async_rpush_pipeline.assert_called_once() + rpush_list = mock_redis_cache.async_rpush_pipeline.call_args.kwargs["rpush_list"] + pushed_keys = {op["key"] for op in rpush_list} + assert pushed_keys == { + REDIS_UPDATE_BUFFER_KEY, + REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY, + } + # Payloads round-trip through the same JSON encoding used on the store path + payloads = {op["key"]: json.loads(op["values"][0]) for op in rpush_list} + assert payloads[REDIS_UPDATE_BUFFER_KEY] == db_spend + assert payloads[REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY] == daily_user + + +@pytest.mark.asyncio +async def test_restore_transactions_to_redis_noop_when_empty( + redis_update_buffer, mock_redis_cache +): + """Nothing to restore -> no Redis call.""" + mock_redis_cache.async_rpush_pipeline = AsyncMock() + await redis_update_buffer.restore_transactions_to_redis() + mock_redis_cache.async_rpush_pipeline.assert_not_called() + + def test_validate_redis_transaction_buffer_raises_without_redis(): """ When use_redis_transaction_buffer=true but no Redis cache is configured, diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 4c17c5d3482..10544e82453 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1532,9 +1532,9 @@ async def test_commit_spend_updates_uses_pipeline(): mock_redis_update_buffer = AsyncMock() mock_redis_update_buffer.store_in_memory_spend_updates_in_redis = AsyncMock() - # Return all-None tuple (no data to commit) + # Return all-None tuple (no data to commit); the pipeline yields 6 slots mock_redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline = ( - AsyncMock(return_value=(None, None, None, None, None, None, None)) + AsyncMock(return_value=(None, None, None, None, None, None)) ) db_writer.redis_update_buffer = mock_redis_update_buffer @@ -1565,6 +1565,153 @@ async def test_commit_spend_updates_uses_pipeline(): mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer.assert_not_called() +@pytest.mark.asyncio +async def test_commit_with_redis_requeues_all_on_db_failure(): + """ + Regression for #33872: if the DB commit fails after the leader has already + popped transactions from Redis, the popped transactions must be re-queued to + Redis so a later tick can retry them, instead of being silently lost. + """ + db_writer = DBSpendUpdateWriter() + + db_spend = { + "user_list_transactions": {"user1": 1.5}, + "end_user_list_transactions": {}, + "key_list_transactions": {"key1": 1.5}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + daily_user = {"user_key1": {"spend": 1.5, "api_requests": 1}} + + mock_redis_update_buffer = AsyncMock() + mock_redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline = AsyncMock( + return_value=(db_spend, daily_user, None, None, None, None) + ) + mock_redis_update_buffer.restore_transactions_to_redis = AsyncMock() + db_writer.redis_update_buffer = mock_redis_update_buffer + + mock_pod_lock_manager = AsyncMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + db_writer.pod_lock_manager = mock_pod_lock_manager + + # Every DB write raises -> simulates a full database outage + db_writer._commit_spend_updates_to_db = AsyncMock(side_effect=Exception("db down")) + + with patch.object( + DBSpendUpdateWriter, + "update_daily_user_spend", + new=AsyncMock(side_effect=Exception("db down")), + ): + await db_writer._commit_spend_updates_to_db_with_redis( + prisma_client=MagicMock(), + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + + # Both failed categories must be re-queued to Redis, nothing lost + mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once() + _, kwargs = mock_redis_update_buffer.restore_transactions_to_redis.call_args + assert kwargs["db_spend_update_transactions"] == db_spend + assert kwargs["daily_spend_update_transactions"] == daily_user + # The lock must still be released + mock_pod_lock_manager.release_lock.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_commit_with_redis_only_requeues_failed_category(): + """ + A partial DB failure must not re-queue categories that already committed, + otherwise their spend would be double-counted on the next tick. + """ + db_writer = DBSpendUpdateWriter() + + db_spend = { + "user_list_transactions": {"user1": 1.5}, + "end_user_list_transactions": {}, + "key_list_transactions": {}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + daily_user = {"user_key1": {"spend": 1.5, "api_requests": 1}} + + mock_redis_update_buffer = AsyncMock() + mock_redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline = AsyncMock( + return_value=(db_spend, daily_user, None, None, None, None) + ) + mock_redis_update_buffer.restore_transactions_to_redis = AsyncMock() + db_writer.redis_update_buffer = mock_redis_update_buffer + + mock_pod_lock_manager = AsyncMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + db_writer.pod_lock_manager = mock_pod_lock_manager + + # db_spend commits fine; only the daily user commit fails + db_writer._commit_spend_updates_to_db = AsyncMock() + + with patch.object( + DBSpendUpdateWriter, + "update_daily_user_spend", + new=AsyncMock(side_effect=Exception("db down")), + ): + await db_writer._commit_spend_updates_to_db_with_redis( + prisma_client=MagicMock(), + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + + mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once() + _, kwargs = mock_redis_update_buffer.restore_transactions_to_redis.call_args + # Only the failed daily category is requeued; the committed db_spend is not + assert kwargs == {"daily_spend_update_transactions": daily_user} + + +@pytest.mark.asyncio +async def test_commit_with_redis_no_requeue_on_success(): + """When all commits succeed, nothing should be re-queued to Redis.""" + db_writer = DBSpendUpdateWriter() + + db_spend = { + "user_list_transactions": {"user1": 1.5}, + "end_user_list_transactions": {}, + "key_list_transactions": {}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + + mock_redis_update_buffer = AsyncMock() + mock_redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline = AsyncMock( + return_value=(db_spend, None, None, None, None, None) + ) + mock_redis_update_buffer.restore_transactions_to_redis = AsyncMock() + db_writer.redis_update_buffer = mock_redis_update_buffer + + mock_pod_lock_manager = AsyncMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + db_writer.pod_lock_manager = mock_pod_lock_manager + + db_writer._commit_spend_updates_to_db = AsyncMock() + + await db_writer._commit_spend_updates_to_db_with_redis( + prisma_client=MagicMock(), + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + + mock_redis_update_buffer.restore_transactions_to_redis.assert_not_awaited() + + @pytest.mark.parametrize( "bucket_name,input_dict,table_attr,method_name,where_key,expected_order", [ From 118b47a8a39a79922f09c851364e43e3c73d0ce1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 23:45:23 +0000 Subject: [PATCH 002/672] fix(proxy): keep tag drain inside try and cover requeue paths with tests Move the destructive daily-tag Redis drain back inside the try so a Redis read failure still releases the pod lock via the finally block, and use a covariant Mapping for the restore signature. Add regression tests for the daily-tag requeue-on-failure/no-requeue-on-success paths and the RedisError swallow branch in restore_transactions_to_redis. --- litellm/proxy/db/db_spend_update_writer.py | 13 ++-- .../redis_update_buffer.py | 13 ++-- .../test_redis_update_buffer.py | 18 +++++ .../proxy/db/test_db_spend_update_writer.py | 72 +++++++++++++++++++ 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index cc266019ff7..f13bf2e2105 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -797,11 +797,7 @@ class DBSpendUpdateWriter: ): verbose_proxy_logger.debug("acquired lock for spend updates") - # Track everything popped from Redis. Each category is removed once it - # has been committed to the DB, so whatever is left after a failure can - # be re-queued for the next tick instead of being lost. Committed - # categories are never re-queued, so their spend is not double-counted. - uncommitted: dict[str, Any] = {} # mutable-ok: drives which popped categories still need re-queuing + uncommitted: dict[str, Any] = {} # mutable-ok: tracks popped categories still needing commit try: ( @@ -1048,11 +1044,12 @@ class DBSpendUpdateWriter: cronjob_id=DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, ): verbose_proxy_logger.debug("acquired lock for daily tag spend updates") - daily_tag_spend_update_transactions = ( - await self.redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() - ) + daily_tag_spend_update_transactions: dict[str, DailyTagSpendTransaction] | None = None committed = False try: + daily_tag_spend_update_transactions = ( + await self.redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + ) if daily_tag_spend_update_transactions: await DBSpendUpdateWriter.update_daily_tag_spend( n_retry_times=n_retry_times, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index b30fadd86ab..660fd514d99 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -6,6 +6,7 @@ This is to prevent deadlocks and improve reliability import asyncio import json +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from redis.exceptions import RedisError @@ -379,12 +380,12 @@ class RedisUpdateBuffer: async def restore_transactions_to_redis( self, db_spend_update_transactions: DBSpendUpdateTransactions | None = None, - daily_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, - daily_team_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, - daily_org_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, - daily_end_user_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, - daily_agent_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, - daily_tag_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None = None, + daily_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, + daily_team_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, + daily_org_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, + daily_end_user_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, + daily_agent_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, + daily_tag_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, ) -> None: """ Re-push transactions that were popped from Redis but not committed to the DB. diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 79909561683..3325893c5f6 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -316,6 +316,24 @@ async def test_restore_transactions_to_redis_noop_when_empty( mock_redis_cache.async_rpush_pipeline.assert_not_called() +@pytest.mark.asyncio +async def test_restore_transactions_to_redis_swallows_redis_error( + redis_update_buffer, mock_redis_cache +): + """A Redis failure during restore must not propagate to the caller's finally block.""" + from redis.exceptions import RedisError + + mock_redis_cache.async_rpush_pipeline = AsyncMock( + side_effect=RedisError("redis down") + ) + + await redis_update_buffer.restore_transactions_to_redis( + db_spend_update_transactions={"key_list_transactions": {"key1": 1.0}}, + ) + + mock_redis_cache.async_rpush_pipeline.assert_called_once() + + def test_validate_redis_transaction_buffer_raises_without_redis(): """ When use_redis_transaction_buffer=true but no Redis cache is configured, diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 10544e82453..06d06b50234 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1712,6 +1712,78 @@ async def test_commit_with_redis_no_requeue_on_success(): mock_redis_update_buffer.restore_transactions_to_redis.assert_not_awaited() +@pytest.mark.asyncio +async def test_commit_daily_tag_spend_requeues_on_db_failure(): + """A failed daily tag commit must re-queue the popped tag transactions and release the lock.""" + db_writer = DBSpendUpdateWriter() + + daily_tag = {"tag_key1": {"spend": 1.5, "api_requests": 1}} + + mock_redis_update_buffer = AsyncMock() + mock_redis_update_buffer.store_in_memory_daily_tag_spend_updates_in_redis = AsyncMock() + mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer = AsyncMock( + return_value=daily_tag + ) + mock_redis_update_buffer.restore_transactions_to_redis = AsyncMock() + db_writer.redis_update_buffer = mock_redis_update_buffer + + mock_pod_lock_manager = AsyncMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + db_writer.pod_lock_manager = mock_pod_lock_manager + + with patch.object( + DBSpendUpdateWriter, + "update_daily_tag_spend", + new=AsyncMock(side_effect=Exception("db down")), + ): + await db_writer._commit_daily_tag_spend_to_db_with_redis( + prisma_client=MagicMock(), + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + + mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once_with( + daily_tag_spend_update_transactions=daily_tag, + ) + mock_pod_lock_manager.release_lock.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_commit_daily_tag_spend_no_requeue_on_success(): + """A successful daily tag commit must not re-queue anything.""" + db_writer = DBSpendUpdateWriter() + + daily_tag = {"tag_key1": {"spend": 1.5, "api_requests": 1}} + + mock_redis_update_buffer = AsyncMock() + mock_redis_update_buffer.store_in_memory_daily_tag_spend_updates_in_redis = AsyncMock() + mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer = AsyncMock( + return_value=daily_tag + ) + mock_redis_update_buffer.restore_transactions_to_redis = AsyncMock() + db_writer.redis_update_buffer = mock_redis_update_buffer + + mock_pod_lock_manager = AsyncMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + db_writer.pod_lock_manager = mock_pod_lock_manager + + with patch.object( + DBSpendUpdateWriter, + "update_daily_tag_spend", + new=AsyncMock(), + ): + await db_writer._commit_daily_tag_spend_to_db_with_redis( + prisma_client=MagicMock(), + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + + mock_redis_update_buffer.restore_transactions_to_redis.assert_not_awaited() + mock_pod_lock_manager.release_lock.assert_awaited_once() + + @pytest.mark.parametrize( "bucket_name,input_dict,table_attr,method_name,where_key,expected_order", [ From ab997e04eb4f0f50bc2c6ae738231455cdd97329 Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 25 Jul 2026 00:09:21 +0000 Subject: [PATCH 003/672] fix(caching): cache anthropic /v1/messages responses, including streaming anthropic_messages was missing from the cache's supported call types, so every /v1/messages request went to the provider. Adding it alone is not enough: the cache key is built from the OpenAI-ish param set, which has no system, top_k or stop_sequences, so two requests differing only by system prompt shared an entry and the second got the first one's answer. The Anthropic Messages request shape now feeds the key set as well. Streaming responses return to the caller before async_set_cache runs, so they are teed on the way out and the SSE events are stored verbatim once the stream reaches message_stop without a provider error. A hit replays those bytes and logs the request as a cache hit with zero cost. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching.py | 45 +---- litellm/caching/caching_handler.py | 41 +++- .../litellm_core_utils/model_param_helper.py | 19 +- .../messages/response_cache.py | 163 ++++++++++++++++ .../anthropic_passthrough_logging_handler.py | 16 +- .../streaming_handler.py | 2 +- litellm/types/caching.py | 19 ++ litellm/utils.py | 5 +- tests/test_litellm/caching/test_caching.py | 21 ++ .../messages/test_response_cache.py | 179 ++++++++++++++++++ 10 files changed, 457 insertions(+), 53 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 34badaa3e8a..88a5e08604e 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -67,20 +67,7 @@ class Cache: default_in_memory_ttl: Optional[float] = None, default_in_redis_ttl: Optional[float] = None, similarity_threshold: Optional[float] = None, - supported_call_types: Optional[List[CachingSupportedCallTypes]] = [ - "completion", - "acompletion", - "embedding", - "aembedding", - "atranscription", - "transcription", - "atext_completion", - "text_completion", - "arerank", - "rerank", - "responses", - "aresponses", - ], + supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES), # s3 Bucket, boto3 configuration azure_account_url: Optional[str] = None, azure_blob_container: Optional[str] = None, @@ -930,20 +917,7 @@ def enable_cache( host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, - supported_call_types: Optional[List[CachingSupportedCallTypes]] = [ - "completion", - "acompletion", - "embedding", - "aembedding", - "atranscription", - "transcription", - "atext_completion", - "text_completion", - "arerank", - "rerank", - "responses", - "aresponses", - ], + supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES), **kwargs, ): """ @@ -990,20 +964,7 @@ def update_cache( host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, - supported_call_types: Optional[List[CachingSupportedCallTypes]] = [ - "completion", - "acompletion", - "embedding", - "aembedding", - "atranscription", - "transcription", - "atext_completion", - "text_completion", - "arerank", - "rerank", - "responses", - "aresponses", - ], + supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES), **kwargs, ): """ diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index b17e055c7ea..8b2d033f24a 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -116,7 +116,8 @@ def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bo When stream=True, do not run success callbacks at cache-hit time. Cached chat/text completion replay uses CustomStreamWrapper; cached Responses - replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success + replay uses CachedResponsesAPIStreamingIterator; cached Anthropic Messages + replay uses CachedAnthropicMessagesStreamIterator. All invoke logging success handlers when the stream finishes; firing them here too would double-count spend and callback records. """ @@ -848,6 +849,18 @@ class LLMCachingHandler: response_type="audio_transcription", hidden_params=hidden_params, ) + elif ( + call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value + ) and isinstance(cached_result, dict): + from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import ( + convert_cached_anthropic_messages_result, + ) + + cached_result = convert_cached_anthropic_messages_result( + cached_result=cached_result, + logging_obj=logging_obj, + kwargs=kwargs, + ) elif (call_type == "aresponses" or call_type == "responses") and isinstance(cached_result, dict): use_chat_completion_cache = _is_chat_completion_cached_dict(cached_result) if use_chat_completion_cache: @@ -1044,6 +1057,32 @@ class LLMCachingHandler: and (kwargs.get("cache", {}).get("no-store", False) is not True) ) + def wrap_streaming_result_for_cache(self, result: Any, call_type: str) -> Any: + """ + Tee a streaming result so it still reaches the cache. + + Streaming responses are returned to the caller before ``async_set_cache`` + runs. Chat/text completion streams are teed inside ``CustomStreamWrapper`` + and Responses API streams inside their own iterator; Anthropic Messages + streams have no such hook, so they are wrapped here. + """ + if call_type not in ( + CallTypes.anthropic_messages.value, + CallTypes.aanthropic_messages.value, + ): + return result + if litellm.cache is None or not self._should_store_result_in_cache( + original_function=self.original_function, kwargs=self.request_kwargs + ): + return result + if not hasattr(result, "__anext__"): + return result + from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import ( + AnthropicMessagesStreamCacheWriter, + ) + + return AnthropicMessagesStreamCacheWriter(stream=result, caching_handler=self) + def _is_call_type_supported_by_cache( self, original_function: Callable, diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index 39b3f0d5376..cf4eba933b8 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -18,6 +18,7 @@ from openai.types.responses.response_create_params import ( ) from litellm._logging import verbose_logger +from litellm.types.llms.anthropic import AnthropicMessagesRequest from litellm.types.rerank import RerankRequest @@ -40,7 +41,7 @@ class ModelParamHelper: @staticmethod def get_exclude_params_for_model_parameters() -> Set[str]: - return set(["messages", "prompt", "input"]) + return set(["messages", "prompt", "input", "system"]) @staticmethod def _get_relevant_args_to_use_for_logging() -> Set[str]: @@ -73,6 +74,7 @@ class ModelParamHelper: transcription_kwargs = ModelParamHelper._get_litellm_supported_transcription_kwargs() rerank_kwargs = ModelParamHelper._get_litellm_supported_rerank_kwargs() responses_api_kwargs = ModelParamHelper._get_litellm_supported_responses_api_kwargs() + anthropic_messages_kwargs = ModelParamHelper._get_litellm_supported_anthropic_messages_kwargs() exclude_kwargs = ModelParamHelper._get_exclude_kwargs() combined_kwargs = chat_completion_kwargs.union( @@ -81,6 +83,7 @@ class ModelParamHelper: transcription_kwargs, rerank_kwargs, responses_api_kwargs, + anthropic_messages_kwargs, ) combined_kwargs = combined_kwargs.difference(exclude_kwargs) return combined_kwargs @@ -167,12 +170,24 @@ class ModelParamHelper: streaming_params: Set[str] = set(getattr(ResponseCreateParamsStreaming, "__annotations__", {}).keys()) return non_streaming_params.union(streaming_params) + @staticmethod + def _get_litellm_supported_anthropic_messages_kwargs() -> set[str]: + """ + Get the litellm supported Anthropic /v1/messages kwargs + + This follows the Anthropic Messages API spec. `system`, `top_k` and + `stop_sequences` have no OpenAI equivalent, so without them the cache key + for a /v1/messages request ignores them and collides across requests that + differ only by system prompt. + """ + return set(getattr(AnthropicMessagesRequest, "__annotations__", {}).keys()) + @staticmethod def _get_exclude_kwargs() -> Set[str]: """ Get the kwargs to exclude from the cache key """ - return set(["metadata"]) + return set(["metadata", "litellm_metadata"]) ModelParamHelper._relevant_logging_args = frozenset(ModelParamHelper._get_relevant_args_to_use_for_logging()) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py new file mode 100644 index 00000000000..e94d8f6bbaf --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py @@ -0,0 +1,163 @@ +""" +Response caching for Anthropic Messages (`/v1/messages`) requests. + +Non-streaming responses are plain dicts and are stored by the generic caching +handler. Streaming responses are returned to the caller before +``LLMCachingHandler.async_set_cache`` runs, so they are teed here instead: the +SSE events are buffered while they are forwarded and persisted verbatim once the +stream completes, and a hit replays exactly what the provider sent. +""" + +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any, cast + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + BaseAnthropicMessagesStreamingIterator, + _is_message_stop_chunk, + _is_provider_error_chunk, + aclose_if_supported, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) + +if TYPE_CHECKING: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LLMCachingHandler = Any + LiteLLMLoggingObj = Any + +CACHED_STREAM_EVENTS_KEY = "litellm_cached_anthropic_sse_events" + + +def _decode(chunk: bytes | str) -> str: + return chunk.decode("utf-8") if isinstance(chunk, bytes) else chunk + + +class AnthropicMessagesStreamCacheWriter: + """ + Forwards a `/v1/messages` SSE stream unchanged while buffering it, then + writes the collected events to the response cache on normal completion. + + Only a stream that ran to a ``message_stop`` without a provider ``error`` + event is written, so partial or failed responses cannot be replayed. + """ + + def __init__( + self, + stream: AsyncIterator[bytes | str], + caching_handler: "LLMCachingHandler", + ) -> None: + self.stream = stream + self.caching_handler = caching_handler + self.collected_events: list[str] = [] + self.saw_message_stop = False + self.saw_provider_error = False + self.persisted = False + self._hidden_params: dict[str, Any] = getattr(stream, "_hidden_params", {}) or {} + + def __aiter__(self) -> "AnthropicMessagesStreamCacheWriter": + return self + + async def __anext__(self) -> bytes | str: + try: + chunk = await self.stream.__anext__() + except StopAsyncIteration: + await self._persist() + raise + chunk_bytes = chunk.encode("utf-8") if isinstance(chunk, str) else chunk + self.saw_message_stop = self.saw_message_stop or _is_message_stop_chunk(chunk_bytes) + self.saw_provider_error = self.saw_provider_error or _is_provider_error_chunk(chunk_bytes) + self.collected_events.append(_decode(chunk)) + return chunk + + async def aclose(self) -> None: + await aclose_if_supported(self.stream) + + async def _persist(self) -> None: + if self.persisted or litellm.cache is None: + return + if not self.saw_message_stop or self.saw_provider_error: + return + self.persisted = True + + request_kwargs = dict(self.caching_handler.request_kwargs) + if not self.caching_handler._should_store_result_in_cache( + original_function=self.caching_handler.original_function, + kwargs=request_kwargs, + ): + return + preset_cache_key = self.caching_handler.preset_cache_key + if preset_cache_key is not None: + request_kwargs["cache_key"] = preset_cache_key + + try: + await litellm.cache.async_add_cache( + {CACHED_STREAM_EVENTS_KEY: self.collected_events}, + dynamic_cache_object=self.caching_handler.dual_cache, + **request_kwargs, + ) + except Exception as e: + verbose_logger.exception("Anthropic Messages stream cache write failed: %s", e) + + +class CachedAnthropicMessagesStreamIterator(BaseAnthropicMessagesStreamingIterator): + """ + Replays cached `/v1/messages` SSE events and logs the request as a cache hit + once the replay finishes, mirroring what the live stream logs at end of stream. + """ + + def __init__( + self, + events: list[str], + litellm_logging_obj: LiteLLMLoggingObj, + request_body: dict[str, Any], + ) -> None: + super().__init__(litellm_logging_obj=litellm_logging_obj, request_body=request_body) + self.chunks: list[bytes] = [event.encode("utf-8") for event in events] + self.current_index = 0 + self._hidden_params: dict[str, Any] = {"cache_hit": True} + litellm_logging_obj.model_call_details["cache_hit"] = True + + def __aiter__(self) -> "CachedAnthropicMessagesStreamIterator": + return self + + async def __anext__(self) -> bytes: + if self.current_index >= len(self.chunks): + await self._handle_streaming_logging(self.chunks) + raise StopAsyncIteration + chunk = self.chunks[self.current_index] + self.current_index += 1 + return chunk + + +def get_cached_stream_events(cached_result: dict[str, Any]) -> list[str] | None: + events = cached_result.get(CACHED_STREAM_EVENTS_KEY) + if isinstance(events, list): + return [_decode(event) for event in events] + return None + + +def convert_cached_anthropic_messages_result( + cached_result: dict[str, Any], + logging_obj: LiteLLMLoggingObj, + kwargs: dict[str, Any], +) -> AnthropicMessagesResponse | CachedAnthropicMessagesStreamIterator: + """ + Turn a cached `/v1/messages` entry back into what the caller expects: an + SSE replay iterator for a streamed entry, otherwise the response itself + (``AnthropicMessagesResponse`` is a TypedDict, i.e. a dict at runtime). + """ + events = get_cached_stream_events(cached_result) + if events is not None: + return CachedAnthropicMessagesStreamIterator( + events=events, + litellm_logging_obj=logging_obj, + request_body=kwargs, + ) + return cast( # cast-ok: AnthropicMessagesResponse is a TypedDict; validating would drop provider fields we must replay verbatim + AnthropicMessagesResponse, cached_result + ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 50e90699194..51813983876 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -255,12 +255,16 @@ class AnthropicPassthroughLoggingHandler: litellm_params=(logging_obj.litellm_params if hasattr(logging_obj, "litellm_params") else None) ) - response_cost = litellm.completion_cost( - completion_response=litellm_model_response, - model=model_for_cost, - custom_llm_provider=custom_llm_provider, - custom_pricing=custom_pricing, - router_model_id=router_model_id, + response_cost = ( + 0.0 + if logging_obj.model_call_details.get("cache_hit") is True + else litellm.completion_cost( + completion_response=litellm_model_response, + model=model_for_cost, + custom_llm_provider=custom_llm_provider, + custom_pricing=custom_pricing, + router_model_id=router_model_id, + ) ) kwargs["response_cost"] = response_cost diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 4dc1e0e70dd..24e5f1d16d5 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -161,7 +161,7 @@ class PassThroughStreamingHandler: result=standard_logging_response_object, start_time=start_time, end_time=end_time, - cache_hit=False, + cache_hit=litellm_logging_obj.model_call_details.get("cache_hit") is True, prefer_async_handlers=True, **kwargs, ) diff --git a/litellm/types/caching.py b/litellm/types/caching.py index eaa80c2f525..4255a8bd7fc 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -30,8 +30,27 @@ CachingSupportedCallTypes = Literal[ "rerank", "responses", "aresponses", + "anthropic_messages", + "aanthropic_messages", ] +DEFAULT_CACHING_SUPPORTED_CALL_TYPES: tuple[CachingSupportedCallTypes, ...] = ( + "completion", + "acompletion", + "embedding", + "aembedding", + "atranscription", + "transcription", + "atext_completion", + "text_completion", + "arerank", + "rerank", + "responses", + "aresponses", + "anthropic_messages", + "aanthropic_messages", +) + class RedisPipelineIncrementOperation(TypedDict): """ diff --git a/litellm/utils.py b/litellm/utils.py index a11c5500503..f5c8330c284 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1708,7 +1708,10 @@ def client(original_function): start_time=start_time, end_time=end_time, ) - return result + return _llm_caching_handler.wrap_streaming_result_for_cache( + result=result, + call_type=call_type, + ) elif call_type == CallTypes.arealtime.value: return result ### POST-CALL RULES ### diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index eaee54bac5a..b65e8773c85 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -1,6 +1,8 @@ import logging import re +import pytest + from litellm.caching.caching import Cache from litellm.types.caching import LiteLLMCacheType from litellm.types.utils import Embedding, EmbeddingResponse, Usage @@ -146,3 +148,22 @@ def test_exact_cache_key_still_includes_prompt(): model="gpt-4o-mini", messages=[{"role": "user", "content": "b"}] ) assert key_a != key_b + + +@pytest.mark.parametrize( + "anthropic_param", + [ + {"system": "answer ALPHA"}, + {"top_k": 5}, + {"stop_sequences": ["STOP"]}, + ], +) +def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): + """Anthropic /v1/messages params with no OpenAI equivalent must still key the + cache; without them two requests that differ only by system prompt collide.""" + cache = Cache(type=LiteLLMCacheType.LOCAL) + messages = [{"role": "user", "content": "which greek letter?"}] + baseline = cache.get_cache_key(model="claude-sonnet-4-5", messages=messages) + assert baseline != cache.get_cache_key( + model="claude-sonnet-4-5", messages=messages, **anthropic_param + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py new file mode 100644 index 00000000000..344152cd828 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py @@ -0,0 +1,179 @@ +import asyncio +import os +import sys +from typing import Any, AsyncIterator, Dict, List + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +import litellm +from litellm.caching.caching import Cache, LiteLLMCacheType +from litellm.llms.anthropic.experimental_pass_through.messages import handler + +STREAM_EVENTS: List[bytes] = [ + b'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_stream_1", "type": "message", ' + b'"role": "assistant", "model": "claude-sonnet-4-5", "content": [], "stop_reason": null, ' + b'"usage": {"input_tokens": 10, "output_tokens": 0}}}\n\n', + b'event: content_block_start\ndata: {"type": "content_block_start", "index": 0, ' + b'"content_block": {"type": "text", "text": ""}}\n\n', + b'event: content_block_delta\ndata: {"type": "content_block_delta", "index": 0, ' + b'"delta": {"type": "text_delta", "text": "ALPHA"}}\n\n', + b'event: content_block_stop\ndata: {"type": "content_block_stop", "index": 0}\n\n', + b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, ' + b'"usage": {"output_tokens": 3}}\n\n', + b'event: message_stop\ndata: {"type": "message_stop"}\n\n', +] + + +def _anthropic_response(message_id: str, text: str) -> Dict[str, Any]: + return { + "id": message_id, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 3}, + } + + +class _CountingHandler: + """Stands in for the provider dispatch so cache hits are observable as skipped calls.""" + + def __init__(self, results: List[Any]) -> None: + self.results = results + self.calls: List[Dict[str, Any]] = [] + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + self.calls.append(kwargs) + return self.results[min(len(self.calls) - 1, len(self.results) - 1)] + + +async def _byte_stream(chunks: List[bytes]) -> AsyncIterator[bytes]: + for chunk in chunks: + yield chunk + + +async def _collect(stream: AsyncIterator[bytes]) -> List[bytes]: + return [chunk async for chunk in stream] + + +@pytest.fixture +def local_cache(): + previous_cache = litellm.cache + litellm.cache = Cache(type=LiteLLMCacheType.LOCAL) + yield litellm.cache + litellm.cache = previous_cache + + +@pytest.fixture +def request_kwargs() -> Dict[str, Any]: + return { + "model": "anthropic/claude-sonnet-4-5", + "custom_llm_provider": "anthropic", + "api_key": "fake-key", + "max_tokens": 64, + "messages": [{"role": "user", "content": "which greek letter?"}], + } + + +@pytest.mark.asyncio +async def test_non_streaming_request_is_served_from_cache(local_cache, request_kwargs, monkeypatch): + fake_handler = _CountingHandler([_anthropic_response("msg_1", "ALPHA"), _anthropic_response("msg_2", "BETA")]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + first = await litellm.anthropic_messages(**request_kwargs) + await asyncio.sleep(0) + second = await litellm.anthropic_messages(**request_kwargs) + + assert len(fake_handler.calls) == 1 + assert first == second + assert second["content"][0]["text"] == "ALPHA" + + +@pytest.mark.asyncio +async def test_cache_key_separates_different_system_prompts(local_cache, request_kwargs, monkeypatch): + """`system` has no OpenAI equivalent; if it is dropped from the cache key the + second request is answered with the first system prompt's response.""" + fake_handler = _CountingHandler([_anthropic_response("msg_1", "ALPHA"), _anthropic_response("msg_2", "BETA")]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + first = await litellm.anthropic_messages(**request_kwargs, system="Always answer ALPHA") + await asyncio.sleep(0) + second = await litellm.anthropic_messages(**request_kwargs, system="Always answer BETA") + + assert len(fake_handler.calls) == 2 + assert first["content"][0]["text"] == "ALPHA" + assert second["content"][0]["text"] == "BETA" + + +@pytest.mark.parametrize("anthropic_param", [{"top_k": 5}, {"stop_sequences": ["STOP"]}]) +@pytest.mark.asyncio +async def test_cache_key_separates_anthropic_native_params(local_cache, request_kwargs, monkeypatch, anthropic_param): + fake_handler = _CountingHandler([_anthropic_response("msg_1", "ALPHA"), _anthropic_response("msg_2", "BETA")]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + await litellm.anthropic_messages(**request_kwargs) + await asyncio.sleep(0) + await litellm.anthropic_messages(**request_kwargs, **anthropic_param) + + assert len(fake_handler.calls) == 2 + + +@pytest.mark.asyncio +async def test_streaming_request_is_replayed_from_cache(local_cache, request_kwargs, monkeypatch): + fake_handler = _CountingHandler([_byte_stream(STREAM_EVENTS), _byte_stream([b"event: never_used\n\n"])]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + first = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + second_stream = await litellm.anthropic_messages(**request_kwargs, stream=True) + second = await _collect(second_stream) + + assert len(fake_handler.calls) == 1 + assert first == STREAM_EVENTS + assert second == STREAM_EVENTS + assert second_stream._hidden_params["cache_hit"] is True + + +@pytest.mark.asyncio +async def test_streaming_cache_is_not_shared_with_non_streaming(local_cache, request_kwargs, monkeypatch): + fake_handler = _CountingHandler([_byte_stream(STREAM_EVENTS), _anthropic_response("msg_2", "ALPHA")]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + non_streaming = await litellm.anthropic_messages(**request_kwargs) + + assert len(fake_handler.calls) == 2 + assert non_streaming["content"][0]["text"] == "ALPHA" + + +@pytest.mark.asyncio +async def test_failed_stream_is_not_cached(local_cache, request_kwargs, monkeypatch): + error_events = STREAM_EVENTS[:3] + [ + b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "overloaded"}}\n\n' + ] + fake_handler = _CountingHandler([_byte_stream(error_events), _byte_stream(STREAM_EVENTS)]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + failed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + + assert failed == error_events + assert len(fake_handler.calls) == 2 + assert replayed == STREAM_EVENTS + + +@pytest.mark.asyncio +async def test_abandoned_stream_is_not_cached(local_cache, request_kwargs, monkeypatch): + fake_handler = _CountingHandler([_byte_stream(STREAM_EVENTS), _byte_stream(STREAM_EVENTS)]) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + partial_stream = await litellm.anthropic_messages(**request_kwargs, stream=True) + await partial_stream.__anext__() + await partial_stream.aclose() + + replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + + assert len(fake_handler.calls) == 2 + assert replayed == STREAM_EVENTS From d2a5de2e04d10042060c1e68c157857bdacfb165 Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 25 Jul 2026 00:15:55 +0000 Subject: [PATCH 004/672] refactor(caching): tighten anthropic messages cache types and drop comments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching_handler.py | 11 ++------ .../litellm_core_utils/model_param_helper.py | 7 +---- .../messages/response_cache.py | 28 ------------------- 3 files changed, 3 insertions(+), 43 deletions(-) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 8b2d033f24a..70a2e3fd1b2 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -18,6 +18,7 @@ import asyncio import datetime import inspect import time +from collections.abc import AsyncIterator from typing import ( TYPE_CHECKING, Any, @@ -1058,14 +1059,6 @@ class LLMCachingHandler: ) def wrap_streaming_result_for_cache(self, result: Any, call_type: str) -> Any: - """ - Tee a streaming result so it still reaches the cache. - - Streaming responses are returned to the caller before ``async_set_cache`` - runs. Chat/text completion streams are teed inside ``CustomStreamWrapper`` - and Responses API streams inside their own iterator; Anthropic Messages - streams have no such hook, so they are wrapped here. - """ if call_type not in ( CallTypes.anthropic_messages.value, CallTypes.aanthropic_messages.value, @@ -1075,7 +1068,7 @@ class LLMCachingHandler: original_function=self.original_function, kwargs=self.request_kwargs ): return result - if not hasattr(result, "__anext__"): + if not isinstance(result, AsyncIterator): return result from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import ( AnthropicMessagesStreamCacheWriter, diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index cf4eba933b8..7e99e5fc5b2 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -174,13 +174,8 @@ class ModelParamHelper: def _get_litellm_supported_anthropic_messages_kwargs() -> set[str]: """ Get the litellm supported Anthropic /v1/messages kwargs - - This follows the Anthropic Messages API spec. `system`, `top_k` and - `stop_sequences` have no OpenAI equivalent, so without them the cache key - for a /v1/messages request ignores them and collides across requests that - differ only by system prompt. """ - return set(getattr(AnthropicMessagesRequest, "__annotations__", {}).keys()) + return set(AnthropicMessagesRequest.__annotations__.keys()) @staticmethod def _get_exclude_kwargs() -> Set[str]: diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py index e94d8f6bbaf..4873b1fdb96 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py @@ -1,13 +1,3 @@ -""" -Response caching for Anthropic Messages (`/v1/messages`) requests. - -Non-streaming responses are plain dicts and are stored by the generic caching -handler. Streaming responses are returned to the caller before -``LLMCachingHandler.async_set_cache`` runs, so they are teed here instead: the -SSE events are buffered while they are forwarded and persisted verbatim once the -stream completes, and a hit replays exactly what the provider sent. -""" - from collections.abc import AsyncIterator from typing import TYPE_CHECKING, Any, cast @@ -38,14 +28,6 @@ def _decode(chunk: bytes | str) -> str: class AnthropicMessagesStreamCacheWriter: - """ - Forwards a `/v1/messages` SSE stream unchanged while buffering it, then - writes the collected events to the response cache on normal completion. - - Only a stream that ran to a ``message_stop`` without a provider ``error`` - event is written, so partial or failed responses cannot be replayed. - """ - def __init__( self, stream: AsyncIterator[bytes | str], @@ -105,11 +87,6 @@ class AnthropicMessagesStreamCacheWriter: class CachedAnthropicMessagesStreamIterator(BaseAnthropicMessagesStreamingIterator): - """ - Replays cached `/v1/messages` SSE events and logs the request as a cache hit - once the replay finishes, mirroring what the live stream logs at end of stream. - """ - def __init__( self, events: list[str], @@ -146,11 +123,6 @@ def convert_cached_anthropic_messages_result( logging_obj: LiteLLMLoggingObj, kwargs: dict[str, Any], ) -> AnthropicMessagesResponse | CachedAnthropicMessagesStreamIterator: - """ - Turn a cached `/v1/messages` entry back into what the caller expects: an - SSE replay iterator for a streamed entry, otherwise the response itself - (``AnthropicMessagesResponse`` is a TypedDict, i.e. a dict at runtime). - """ events = get_cached_stream_events(cached_result) if events is not None: return CachedAnthropicMessagesStreamIterator( From b6cf4066f4e907c03f11065f52f4da149e649128 Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 25 Jul 2026 01:17:43 +0000 Subject: [PATCH 005/672] fix(caching): log cached anthropic stream replay only once Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/response_cache.py | 5 ++- .../messages/test_response_cache.py | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py index 4873b1fdb96..d5b68c99130 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py @@ -96,6 +96,7 @@ class CachedAnthropicMessagesStreamIterator(BaseAnthropicMessagesStreamingIterat super().__init__(litellm_logging_obj=litellm_logging_obj, request_body=request_body) self.chunks: list[bytes] = [event.encode("utf-8") for event in events] self.current_index = 0 + self.logged = False self._hidden_params: dict[str, Any] = {"cache_hit": True} litellm_logging_obj.model_call_details["cache_hit"] = True @@ -104,7 +105,9 @@ class CachedAnthropicMessagesStreamIterator(BaseAnthropicMessagesStreamingIterat async def __anext__(self) -> bytes: if self.current_index >= len(self.chunks): - await self._handle_streaming_logging(self.chunks) + if not self.logged: + self.logged = True + await self._handle_streaming_logging(self.chunks) raise StopAsyncIteration chunk = self.chunks[self.current_index] self.current_index += 1 diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py index 344152cd828..071580347a6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py @@ -177,3 +177,35 @@ async def test_abandoned_stream_is_not_cached(local_cache, request_kwargs, monke assert len(fake_handler.calls) == 2 assert replayed == STREAM_EVENTS + +@pytest.mark.asyncio +async def test_cached_stream_replay_logs_once_when_polled_after_exhaustion(): + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import ( + CachedAnthropicMessagesStreamIterator, + ) + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + iterator = CachedAnthropicMessagesStreamIterator( + events=[event.decode("utf-8") for event in STREAM_EVENTS], + litellm_logging_obj=logging_obj, + request_body={"model": "claude-sonnet-4-5"}, + ) + + with patch.object( + PassThroughStreamingHandler, + "_route_streaming_logging_to_handler", + new=AsyncMock(), + ) as mock_route: + assert await _collect(iterator) == STREAM_EVENTS + for _ in range(2): + with pytest.raises(StopAsyncIteration): + await iterator.__anext__() + await asyncio.sleep(0) + + mock_route.assert_called_once() From e4a047526334dc97a93ef364878b14760e85408d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 25 Jul 2026 10:57:19 -0700 Subject: [PATCH 006/672] test: remove four mirror test files that exercise none of their module A second mutation batch scored the previously unmapped mirror files on current staging. These four generate mutants for the module they are named after, yet no test in the file executes any of them; their test-context coverage lands on generic shared machinery or, for the guardrail translation handler remainder, on no litellm line at all. Eight sibling findings that do exercise a different real module are kept for retargeting instead of removal. --- .../datadog/test_datadog_llm_observability.py | 1195 ----------------- .../guardrail_translation/test_handler.py | 37 - .../test_reasoning_content_transformation.py | 296 ---- tests/test_litellm/test_azure_video_router.py | 53 - 4 files changed, 1581 deletions(-) delete mode 100644 tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py delete mode 100644 tests/test_litellm/llms/pass_through/guardrail_translation/test_handler.py delete mode 100644 tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py delete mode 100644 tests/test_litellm/test_azure_video_router.py diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py deleted file mode 100644 index 1cc3591392b..00000000000 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ /dev/null @@ -1,1195 +0,0 @@ -import asyncio -import os -import sys -from datetime import datetime, timedelta, timezone -from typing import Optional -from unittest.mock import MagicMock, Mock, patch - -import pytest - -# Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) -import litellm -from litellm.integrations.custom_logger import CustomLogger -from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger -from litellm.types.integrations.datadog_llm_obs import ( - DatadogLLMObsInitParams, -) -from litellm.types.utils import ( - StandardLoggingGuardrailInformation, - StandardLoggingHiddenParams, - StandardLoggingMetadata, - StandardLoggingModelInformation, - StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, -) - - -def create_standard_logging_payload_with_cache() -> StandardLoggingPayload: - """Create a real StandardLoggingPayload object for testing""" - return StandardLoggingPayload( - id="test-request-id-456", - call_type="completion", - response_cost=0.05, - response_cost_failure_debug_info=None, - status="success", - total_tokens=30, - prompt_tokens=10, - completion_tokens=20, - startTime=1234567890.0, - endTime=1234567891.0, - completionStartTime=1234567890.5, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-4", model_map_value=None - ), - model="gpt-4", - model_id="model-123", - model_group="openai-gpt", - api_base="https://api.openai.com", - metadata=StandardLoggingMetadata( - user_api_key_hash="test_hash", - user_api_key_org_id=None, - user_api_key_alias="test_alias", - user_api_key_team_id="test_team", - user_api_key_user_id="test_user", - user_api_key_team_alias="test_team_alias", - spend_logs_metadata=None, - requester_ip_address="127.0.0.1", - requester_metadata=None, - ), - cache_hit=True, - cache_key="test-cache-key-789", - saved_cache_cost=0.02, - request_tags=[], - end_user=None, - requester_ip_address="127.0.0.1", - messages=[{"role": "user", "content": "Hello, world!"}], - response={"choices": [{"message": {"content": "Hi there!"}}]}, - error_str=None, - model_parameters={"stream": True}, - hidden_params=StandardLoggingHiddenParams( - model_id="model-123", - cache_key="test-cache-key-789", - api_base="https://api.openai.com", - response_cost="0.05", - additional_headers=None, - ), - trace_id="test-trace-id-123", - custom_llm_provider="openai", - ) - - -def create_standard_logging_payload_with_failure() -> StandardLoggingPayload: - """Create a StandardLoggingPayload object for failure testing""" - return StandardLoggingPayload( - id="test-request-id-failure-789", - call_type="completion", - response_cost=0.0, - response_cost_failure_debug_info=None, - status="failure", - total_tokens=0, - prompt_tokens=10, - completion_tokens=0, - startTime=1234567890.0, - endTime=1234567891.0, - completionStartTime=1234567890.5, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-4", model_map_value=None - ), - model="gpt-4", - model_id="model-123", - model_group="openai-gpt", - api_base="https://api.openai.com", - metadata=StandardLoggingMetadata( - user_api_key_hash="test_hash", - user_api_key_org_id=None, - user_api_key_alias="test_alias", - user_api_key_team_id="test_team", - user_api_key_user_id="test_user", - user_api_key_team_alias="test_team_alias", - spend_logs_metadata=None, - requester_ip_address="127.0.0.1", - requester_metadata=None, - ), - cache_hit=False, - cache_key=None, - saved_cache_cost=0.0, - request_tags=[], - end_user=None, - requester_ip_address="127.0.0.1", - messages=[{"role": "user", "content": "Hello, world!"}], - response=None, - error_str="RateLimitError: You exceeded your current quota", - error_information=StandardLoggingPayloadErrorInformation( - error_code="rate_limit_exceeded", - error_class="RateLimitError", - llm_provider="openai", - traceback="Traceback (most recent call last):\n File test.py, line 1\n RateLimitError: You exceeded your current quota", - error_message="RateLimitError: You exceeded your current quota", - ), - model_parameters={"stream": False}, - hidden_params=StandardLoggingHiddenParams( - model_id="model-123", - cache_key=None, - api_base="https://api.openai.com", - response_cost="0.0", - additional_headers=None, - ), - trace_id="test-trace-id-failure-456", - custom_llm_provider="openai", - ) - - -class TestDataDogLLMObsLogger: - """Test suite for DataDog LLM Observability Logger""" - - @pytest.fixture - def mock_env_vars(self): - """Mock environment variables for DataDog""" - with patch.dict( - os.environ, {"DD_API_KEY": "test_api_key", "DD_SITE": "us5.datadoghq.com"} - ): - yield - - @pytest.fixture - def mock_response_obj(self): - """Create a mock response object""" - mock_response = Mock() - mock_response.__getitem__ = Mock( - return_value={ - "choices": [ - { - "message": Mock( - json=Mock( - return_value={"role": "assistant", "content": "Hello!"} - ) - ) - } - ] - } - ) - return mock_response - - def test_cost_and_trace_id_integration(self, mock_env_vars, mock_response_obj): - """Test that total_cost is passed and trace_id from standard payload is used""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_cache() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": { - "metadata": {"trace_id": "old-trace-id-should-be-ignored"} - }, - } - - start_time = datetime.now() - end_time = datetime.now() - - payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) - - # Test 1: Verify total_cost is correctly extracted from response_cost - assert payload["metrics"].get("total_cost") == 0.05 - - # Test 2: Verify trace_id comes from standard_logging_payload, not metadata - assert payload["trace_id"] == "test-trace-id-123" - - # Test 3: Verify saved_cache_cost is in metadata - metadata = payload["meta"]["metadata"] - assert metadata["saved_cache_cost"] == 0.02 - assert metadata["cache_hit"] is True - assert metadata["cache_key"] == "test-cache-key-789" - - # Test 4: Verify is_streamed_request is in metadata - assert metadata["is_streamed_request"] is True - - def test_cache_metadata_fields(self, mock_env_vars, mock_response_obj): - """Test that cache-related metadata fields are correctly tracked""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_cache() - - # Test the _get_dd_llm_obs_payload_metadata method directly - metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) - - # Verify all cache-related fields are present - assert metadata["cache_hit"] is True - assert metadata["cache_key"] == "test-cache-key-789" - assert metadata["saved_cache_cost"] == 0.02 - assert metadata["id"] == "test-request-id-456" - assert metadata["trace_id"] == "test-trace-id-123" - assert metadata["model_name"] == "gpt-4" - assert metadata["model_provider"] == "openai" - - def test_get_time_to_first_token_seconds(self, mock_env_vars): - """Test the _get_time_to_first_token_seconds method for streaming calls""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Test streaming case (completion_start_time available) - streaming_payload = create_standard_logging_payload_with_cache() - # Modify times for testing: start=1000, completion_start=1002, end=1005 - streaming_payload["startTime"] = 1000.0 - streaming_payload["completionStartTime"] = 1002.0 - streaming_payload["endTime"] = 1005.0 - - # Test streaming case: should use completion_start_time - start_time - time_to_first_token = logger._get_time_to_first_token_seconds( - streaming_payload - ) - assert time_to_first_token == 2.0 # 1002.0 - 1000.0 = 2.0 seconds - - def test_datadog_span_kind_mapping(self, mock_env_vars): - """Test that call_type values are correctly mapped to DataDog span kinds""" - from litellm.types.utils import CallTypes - - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Test embedding operations - assert ( - logger._get_datadog_span_kind(CallTypes.embedding.value, "123") - == "embedding" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.aembedding.value, "123") - == "embedding" - ) - - # Test LLM completion operations - assert logger._get_datadog_span_kind(CallTypes.completion.value, None) == "llm" - assert logger._get_datadog_span_kind(CallTypes.acompletion.value, None) == "llm" - assert ( - logger._get_datadog_span_kind(CallTypes.text_completion.value, None) - == "llm" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.generate_content.value, None) - == "llm" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.anthropic_messages.value, None) - == "llm" - ) - assert logger._get_datadog_span_kind(CallTypes.responses.value, None) == "llm" - assert logger._get_datadog_span_kind(CallTypes.aresponses.value, None) == "llm" - - # Test tool operations - assert ( - logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value, "123") - == "tool" - ) - - # Test retrieval operations - assert ( - logger._get_datadog_span_kind(CallTypes.get_assistants.value, "123") - == "retrieval" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.file_retrieve.value, "123") - == "retrieval" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.retrieve_batch.value, "123") - == "retrieval" - ) - - # Test task operations - assert ( - logger._get_datadog_span_kind(CallTypes.create_batch.value, "123") == "task" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.image_generation.value, "123") - == "task" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.moderation.value, "123") == "task" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.transcription.value, "123") - == "task" - ) - - # Test default fallback - assert logger._get_datadog_span_kind("unknown_call_type", None) == "llm" - assert logger._get_datadog_span_kind(None, None) == "llm" - - def test_datadog_span_kind_defaults_without_parent(self, mock_env_vars): - """Test that non-llm kinds fallback to llm when no parent span is provided""" - from litellm.types.utils import CallTypes - - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Tool/task/retrieval span kinds should fallback to llm when parent_id missing - assert ( - logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value, None) == "llm" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.create_batch.value, None) == "llm" - ) - assert ( - logger._get_datadog_span_kind(CallTypes.get_assistants.value, None) == "llm" - ) - - @pytest.mark.asyncio - async def test_async_log_failure_event(self, mock_env_vars): - """Test that async_log_failure_event correctly processes failure payloads according to DD LLM Obs API spec""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Ensure log_queue starts empty - logger.log_queue = [] - - standard_failure_payload = create_standard_logging_payload_with_failure() - - kwargs = { - "standard_logging_object": standard_failure_payload, - "model": "gpt-4", - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() + timedelta(seconds=2) - - # Mock async_send_batch to prevent actual network calls - with patch.object(logger, "async_send_batch") as mock_send_batch: - # Call the method under test - await logger.async_log_failure_event(kwargs, None, start_time, end_time) - - # Verify payload was added to queue - assert len(logger.log_queue) == 1 - - # Verify the payload has correct failure characteristics according to DD LLM Obs API spec - payload = logger.log_queue[0] - assert payload["trace_id"] == "test-trace-id-failure-456" - assert ( - payload["meta"]["metadata"]["id"] == "test-request-id-failure-789" - ) - assert payload["status"] == "error" - - # Verify error information follows DD LLM Obs API spec - assert ( - payload["meta"]["error"]["message"] - == "RateLimitError: You exceeded your current quota" - ) - assert payload["meta"]["error"]["type"] == "RateLimitError" - assert ( - payload["meta"]["error"]["stack"] - == "Traceback (most recent call last):\n File test.py, line 1\n RateLimitError: You exceeded your current quota" - ) - - assert payload["metrics"]["total_cost"] == 0.0 - assert payload["metrics"]["total_tokens"] == 0 - assert payload["metrics"]["output_tokens"] == 0 - - # Verify batch sending not triggered (queue size < batch_size) - mock_send_batch.assert_not_called() - - -class TestDataDogLLMObsLoggerForRedaction(DataDogLLMObsLogger): - """Test suite for DataDog LLM Observability Logger""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.logged_standard_logging_payload: Optional[StandardLoggingPayload] = None - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self.logged_standard_logging_payload = kwargs.get("standard_logging_object") - - -class TestS3Logger(CustomLogger): - """Test suite for S3 Logger""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.logged_standard_logging_payload: Optional[StandardLoggingPayload] = None - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self.logged_standard_logging_payload = kwargs.get("standard_logging_object") - - -@pytest.mark.asyncio -async def test_dd_llms_obs_redaction(mock_env_vars): - # init DD with turn_off_message_logging=True - litellm._turn_on_debug() - from litellm.types.utils import LiteLLMCommonStrings - - litellm.datadog_llm_observability_params = DatadogLLMObsInitParams( - turn_off_message_logging=True - ) - dd_llms_obs_logger = TestDataDogLLMObsLoggerForRedaction() - test_s3_logger = TestS3Logger() - litellm.callbacks = [dd_llms_obs_logger, test_s3_logger] - - # call litellm - await litellm.acompletion( - model="gpt-4o", - mock_response="Hi there!", - messages=[{"role": "user", "content": "Hello, world!"}], - ) - - # sleep 1 second for logging to complete - await asyncio.sleep(1) - - ################# - # test validation - # 1. both loggers logged a standard_logging_payload - # 2. DD LLM Obs standard_logging_payload has messages and response redacted - # 3. S3 standard_logging_payload does not have messages and response redacted - - assert dd_llms_obs_logger.logged_standard_logging_payload is not None - assert test_s3_logger.logged_standard_logging_payload is not None - - assert ( - dd_llms_obs_logger.logged_standard_logging_payload["messages"][0]["content"] - == "redacted-by-litellm" - ) - assert ( - dd_llms_obs_logger.logged_standard_logging_payload["response"]["choices"][0][ - "message" - ]["content"] - == "redacted-by-litellm" - ) - - assert test_s3_logger.logged_standard_logging_payload["messages"] == [ - {"role": "user", "content": "Hello, world!"} - ] - assert ( - test_s3_logger.logged_standard_logging_payload["response"]["choices"][0][ - "message" - ]["content"] - == "Hi there!" - ) - - -@pytest.fixture -def mock_env_vars(): - """Mock environment variables for DataDog""" - with patch.dict( - os.environ, {"DD_API_KEY": "test_api_key", "DD_SITE": "us5.datadoghq.com"} - ): - yield - - -@pytest.mark.asyncio -async def test_create_llm_obs_payload(mock_env_vars): - datadog_llm_obs_logger = DataDogLLMObsLogger() - standard_logging_payload = create_standard_logging_payload_with_cache() - payload = datadog_llm_obs_logger.create_llm_obs_payload( - kwargs={ - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello"}], - "standard_logging_object": standard_logging_payload, - }, - start_time=datetime.now(), - end_time=datetime.now() + timedelta(seconds=1), - ) - - assert payload["name"] == "litellm_llm_call" - assert payload["meta"]["kind"] == "llm" - assert payload["meta"]["input"]["messages"] == [ - {"role": "user", "content": "Hello, world!"} - ] - assert payload["meta"]["output"]["messages"][0]["content"] == "Hi there!" - assert payload["metrics"]["input_tokens"] == 10 - assert payload["metrics"]["output_tokens"] == 20 - assert payload["metrics"]["total_tokens"] == 30 - - -def create_standard_logging_payload_with_latency_metrics() -> StandardLoggingPayload: - """Create a StandardLoggingPayload object with latency metrics for testing""" - guardrail_info = StandardLoggingGuardrailInformation( - guardrail_name="test_guardrail", - guardrail_status="success", - start_time=1234567890.0, - end_time=1234567890.5, - duration=0.5, # 500ms - guardrail_request={"input": "test input message", "user_id": "test_user"}, - guardrail_response={ - "output": "filtered output", - "flagged": False, - "score": 0.1, - }, - ) - - hidden_params = StandardLoggingHiddenParams( - model_id="model-123", - cache_key="test-cache-key", - api_base="https://api.openai.com", - response_cost="0.05", - litellm_overhead_time_ms=150.0, # 150ms - additional_headers=None, - ) - - return StandardLoggingPayload( - id="test-request-id-latency", - call_type="completion", - response_cost=0.05, - response_cost_failure_debug_info=None, - status="success", - total_tokens=30, - prompt_tokens=10, - completion_tokens=20, - startTime=1234567890.0, - endTime=1234567892.0, - completionStartTime=1234567890.8, # 800ms after start - response_time=2.0, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-4", model_map_value=None - ), - model="gpt-4", - model_id="model-123", - model_group="openai-gpt", - api_base="https://api.openai.com", - metadata=StandardLoggingMetadata( - user_api_key_hash="test_hash", - user_api_key_org_id=None, - user_api_key_alias="test_alias", - user_api_key_team_id="test_team", - user_api_key_user_id="test_user", - user_api_key_team_alias="test_team_alias", - spend_logs_metadata=None, - requester_ip_address="127.0.0.1", - requester_metadata=None, - ), - cache_hit=False, - cache_key=None, - saved_cache_cost=0.0, - request_tags=[], - end_user=None, - requester_ip_address="127.0.0.1", - messages=[{"role": "user", "content": "Hello, world!"}], - response={"choices": [{"message": {"content": "Hi there!"}}]}, - error_str=None, - error_information=None, - model_parameters={"stream": True}, - hidden_params=hidden_params, - guardrail_information=[guardrail_info], - trace_id="test-trace-id-latency", - custom_llm_provider="openai", - ) - - -def test_latency_metrics_in_metadata(mock_env_vars): - """Test that time to first token, litellm overhead, and guardrail overhead are included in metadata""" - with ( - patch("litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client"), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_latency_metrics() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() - - # Test the metadata generation directly - metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) - latency_metadata = metadata.get("latency_metrics", {}) - - # Verify time to first token is included (800ms) - assert "time_to_first_token_ms" in latency_metadata - assert ( - abs(latency_metadata["time_to_first_token_ms"] - 800.0) < 0.001 - ) # 0.8 seconds * 1000 with tolerance for floating-point precision - - # Verify litellm overhead is included (150ms) - assert "litellm_overhead_time_ms" in latency_metadata - assert latency_metadata["litellm_overhead_time_ms"] == 150.0 - - # Verify guardrail overhead is included (500ms) - assert "guardrail_overhead_time_ms" in latency_metadata - assert ( - latency_metadata["guardrail_overhead_time_ms"] == 500.0 - ) # 0.5 seconds * 1000 - - # Verify these metrics are also included in the full payload - payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) - payload_metadata_latency = payload["meta"]["metadata"]["latency_metrics"] - - assert abs(payload_metadata_latency["time_to_first_token_ms"] - 800.0) < 0.001 - assert payload_metadata_latency["litellm_overhead_time_ms"] == 150.0 - assert payload_metadata_latency["guardrail_overhead_time_ms"] == 500.0 - - -def test_latency_metrics_edge_cases(mock_env_vars): - """Test latency metrics with edge cases (missing fields, zero values, etc.)""" - with ( - patch("litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client"), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Test case 1: No latency metrics present - standard_payload = create_standard_logging_payload_with_cache() - metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) - - # Should not have latency fields if data is missing/zero - assert "time_to_first_token_ms" not in metadata # Will be 0, so not included - assert ( - "litellm_overhead_time_ms" not in metadata - ) # Not present in hidden_params - assert "guardrail_overhead_time_ms" not in metadata # No guardrail_information - - # Test case 2: Zero time to first token should not be included - standard_payload = create_standard_logging_payload_with_cache() - standard_payload["startTime"] = 1000.0 - standard_payload["completionStartTime"] = 1000.0 # Same time = 0 difference - metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) - assert "time_to_first_token_ms" not in metadata - - # Test case 3: Missing guardrail duration should not crash - standard_payload = create_standard_logging_payload_with_cache() - standard_payload["guardrail_information"] = [ - StandardLoggingGuardrailInformation( - guardrail_name="test", - guardrail_status="success", - # duration is missing - ) - ] - metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) - assert "guardrail_overhead_time_ms" not in metadata - - -def test_guardrail_information_in_metadata(mock_env_vars): - """Test that guardrail_information is included in metadata with input/output fields""" - with ( - patch("litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client"), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Create a standard payload with guardrail information - standard_payload = create_standard_logging_payload_with_latency_metrics() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() - - # Create the payload and verify guardrail_information is in metadata - payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) - metadata = payload["meta"]["metadata"] - - # Verify guardrail_information is present in metadata - assert "guardrail_information" in metadata - assert metadata["guardrail_information"] is not None - - # Verify the guardrail information structure - guardrail_info = metadata["guardrail_information"] - assert guardrail_info[0]["guardrail_name"] == "test_guardrail" - assert guardrail_info[0]["guardrail_status"] == "success" - assert guardrail_info[0]["duration"] == 0.5 - - # Verify input/output fields are present - assert "guardrail_request" in guardrail_info[0] - assert "guardrail_response" in guardrail_info[0] - - # Validate the input/output content - assert guardrail_info[0]["guardrail_request"]["input"] == "test input message" - assert guardrail_info[0]["guardrail_request"]["user_id"] == "test_user" - assert guardrail_info[0]["guardrail_response"]["output"] == "filtered output" - assert guardrail_info[0]["guardrail_response"]["flagged"] is False - assert guardrail_info[0]["guardrail_response"]["score"] == 0.1 - - -def create_standard_logging_payload_with_tool_calls() -> StandardLoggingPayload: - """Create a StandardLoggingPayload object with tool calls for testing""" - return { - "id": "test-request-id-tool-calls", - "trace_id": "test-trace-id-tool-calls", - "call_type": "completion", - "stream": None, - "response_cost": 0.05, - "response_cost_failure_debug_info": None, - "status": "success", - "custom_llm_provider": "openai", - "total_tokens": 50, - "prompt_tokens": 20, - "completion_tokens": 30, - "startTime": 1234567890.0, - "endTime": 1234567891.0, - "completionStartTime": 1234567890.5, - "response_time": 1.0, - "model_map_information": {"model_map_key": "gpt-4", "model_map_value": None}, - "model": "gpt-4", - "model_id": "model-123", - "model_group": "openai-gpt", - "api_base": "https://api.openai.com", - "metadata": { - "user_api_key_hash": "test_hash", - "user_api_key_org_id": None, - "user_api_key_alias": "test_alias", - "user_api_key_team_id": "test_team", - "user_api_key_user_id": "test_user", - "user_api_key_team_alias": "test_team_alias", - "user_api_key_user_email": None, - "user_api_key_end_user_id": None, - "user_api_key_request_route": None, - "spend_logs_metadata": None, - "requester_ip_address": "127.0.0.1", - "requester_metadata": None, - "requester_custom_headers": None, - "prompt_management_metadata": None, - "mcp_tool_call_metadata": None, - "vector_store_request_metadata": None, - "applied_guardrails": None, - "usage_object": None, - "cold_storage_object_key": None, - }, - "cache_hit": False, - "cache_key": None, - "saved_cache_cost": 0.0, - "request_tags": [], - "end_user": None, - "requester_ip_address": "127.0.0.1", - "messages": [ - {"role": "user", "content": "What's the weather?"}, - { - "role": "assistant", - "content": "I'll check the weather for you.", - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"location": "NYC"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_123", - "content": '{"temperature": 72, "condition": "sunny"}', - }, - ], - "response": { - "choices": [ - { - "message": { - "role": "assistant", - "content": "It's 72°F and sunny in NYC!", - "tool_calls": [ - { - "id": "call_456", - "type": "function", - "function": { - "name": "format_response", - "arguments": '{"temp": 72, "condition": "sunny"}', - }, - } - ], - } - } - ] - }, - "error_str": None, - "error_information": None, - "model_parameters": {"temperature": 0.7}, - "hidden_params": { - "model_id": "model-123", - "cache_key": None, - "api_base": "https://api.openai.com", - "response_cost": "0.05", - "litellm_overhead_time_ms": None, - "additional_headers": None, - "batch_models": None, - "litellm_model_name": None, - "usage_object": None, - }, - "guardrail_information": None, - "standard_built_in_tools_params": None, - } # type: ignore - - -class TestDataDogLLMObsLoggerToolCalls: - """Simple test suite for DataDog LLM Observability Logger tool call handling""" - - @pytest.fixture - def mock_env_vars(self): - """Mock environment variables for DataDog""" - with patch.dict( - os.environ, {"DD_API_KEY": "test_api_key", "DD_SITE": "us5.datadoghq.com"} - ): - yield - - def test_tool_call_span_kind_mapping(self, mock_env_vars): - """Test that tool call operations are correctly mapped to 'tool' span kind""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - # Test MCP tool call mapping - from litellm.types.utils import CallTypes - - assert ( - logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value, "123") - == "tool" - ) - - def test_tool_call_payload_creation(self, mock_env_vars): - """Test that tool call payloads are created correctly""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_tool_calls() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() - - payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) - - # Verify basic payload structure - assert payload.get("name") == "litellm_llm_call" - assert payload.get("status") == "ok" - assert ( - payload.get("meta", {}).get("kind") == "llm" - ) # Regular completion, not tool call - - # Verify metrics - metrics = payload.get("metrics", {}) - assert metrics.get("input_tokens") == 20 - assert metrics.get("output_tokens") == 30 - assert metrics.get("total_tokens") == 50 - - def test_tool_call_messages_preserved(self, mock_env_vars): - """Test that tool call messages are preserved in the payload""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_tool_calls() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() - - payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) - - # Verify input messages include tool calls - meta = payload.get("meta", {}) - input_meta = meta.get("input", {}) - input_messages = input_meta.get("messages", []) - assert len(input_messages) == 3 - - # Check assistant message has tool calls - assistant_msg = input_messages[1] - assert assistant_msg.get("role") == "assistant" - assert "tool_calls" in assistant_msg - tool_calls = assistant_msg.get("tool_calls", []) - assert len(tool_calls) == 1 - tool_call = tool_calls[0] - function_info = tool_call.get("function", {}) - assert function_info.get("name") == "get_weather" - - # Check tool message - tool_msg = input_messages[2] - assert tool_msg.get("role") == "tool" - assert tool_msg.get("tool_call_id") == "call_123" - - def test_tool_call_response_handling(self, mock_env_vars): - """Test that tool calls in response are handled correctly""" - with ( - patch( - "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" - ), - patch("asyncio.create_task"), - ): - logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_tool_calls() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() - - payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) - - # Verify output messages include tool calls - meta = payload.get("meta", {}) - output_meta = meta.get("output", {}) - output_messages = output_meta.get("messages", []) - assert len(output_messages) == 1 - - output_msg = output_messages[0] - assert output_msg.get("role") == "assistant" - assert "tool_calls" in output_msg - output_tool_calls = output_msg.get("tool_calls", []) - assert len(output_tool_calls) == 1 - output_function_info = output_tool_calls[0].get("function", {}) - assert output_function_info.get("name") == "format_response" - - -def create_standard_logging_payload_with_spend_metrics() -> StandardLoggingPayload: - """Create a StandardLoggingPayload object with spend metrics for testing""" - from datetime import datetime, timezone - - # Create a budget reset time 10 days from now (using "10d" format) - budget_reset_at = datetime.now(timezone.utc) + timedelta(days=10) - - return { - "id": "test-request-id-spend", - "trace_id": "test-trace-id-spend", - "call_type": "completion", - "stream": None, - "response_cost": 0.15, - "response_cost_failure_debug_info": None, - "status": "success", - "custom_llm_provider": "openai", - "total_tokens": 30, - "prompt_tokens": 10, - "completion_tokens": 20, - "startTime": 1234567890.0, - "endTime": 1234567891.0, - "completionStartTime": 1234567890.5, - "response_time": 1.0, - "model_map_information": {"model_map_key": "gpt-4", "model_map_value": None}, - "model": "gpt-4", - "model_id": "model-123", - "model_group": "openai-gpt", - "api_base": "https://api.openai.com", - "metadata": { - "user_api_key_hash": "test_hash", - "user_api_key_org_id": None, - "user_api_key_alias": "test_alias", - "user_api_key_team_id": "test_team", - "user_api_key_user_id": "test_user", - "user_api_key_team_alias": "test_team_alias", - "user_api_key_user_email": None, - "user_api_key_end_user_id": None, - "user_api_key_request_route": None, - "user_api_key_spend": 0.67, - "user_api_key_max_budget": 10.0, # $10 max budget - "user_api_key_budget_reset_at": budget_reset_at.isoformat(), # ISO format: 2025-09-26T... - "spend_logs_metadata": None, - "requester_ip_address": "127.0.0.1", - "requester_metadata": None, - "requester_custom_headers": None, - "prompt_management_metadata": None, - "mcp_tool_call_metadata": None, - "vector_store_request_metadata": None, - "applied_guardrails": None, - "usage_object": None, - "cold_storage_object_key": None, - }, - "cache_hit": False, - "cache_key": None, - "saved_cache_cost": 0.0, - "request_tags": [], - "end_user": None, - "requester_ip_address": "127.0.0.1", - "messages": [{"role": "user", "content": "Hello, world!"}], - "response": {"choices": [{"message": {"content": "Hi there!"}}]}, - "error_str": None, - "error_information": None, - "model_parameters": {"stream": False}, - "hidden_params": { - "model_id": "model-123", - "cache_key": None, - "api_base": "https://api.openai.com", - "response_cost": "0.15", - "litellm_overhead_time_ms": None, - "additional_headers": None, - "batch_models": None, - "litellm_model_name": None, - "usage_object": None, - }, - "guardrail_information": None, - "standard_built_in_tools_params": None, - } # type: ignore - - -@pytest.mark.asyncio -async def test_datadog_llm_obs_spend_metrics(mock_env_vars): - """Test that budget metrics are properly extracted and logged""" - datadog_llm_obs_logger = DataDogLLMObsLogger() - - # Create a standard logging payload with spend metrics - payload = create_standard_logging_payload_with_spend_metrics() - - # Show the budget reset time in ISO format - budget_reset_iso = payload["metadata"]["user_api_key_budget_reset_at"] - print(f"Budget reset time (ISO format): {budget_reset_iso}") - from datetime import datetime, timezone - - print(f"Current time: {datetime.now(timezone.utc).isoformat()}") - - # Test the _get_spend_metrics method - spend_metrics = datadog_llm_obs_logger._get_spend_metrics(payload) - - # Verify budget metrics are present - assert "user_api_key_max_budget" in spend_metrics - assert spend_metrics["user_api_key_max_budget"] == 10.0 - - assert "user_api_key_budget_reset_at" in spend_metrics - # The budget reset should be a datetime string in ISO format - budget_reset = spend_metrics["user_api_key_budget_reset_at"] - assert isinstance(budget_reset, str) - print(f"Budget reset datetime: {budget_reset}") - # Should be close to 10 days from now - budget_reset_dt = datetime.fromisoformat(budget_reset.replace("Z", "+00:00")) - now = datetime.now(timezone.utc) - time_diff = (budget_reset_dt - now).total_seconds() / 86400 # days - assert 9.5 <= time_diff <= 10.5 # Should be close to 10 days - - print(f"Spend metrics: {spend_metrics}") - - -@pytest.mark.asyncio -async def test_datadog_llm_obs_spend_metrics_no_budget(mock_env_vars): - """Test that spend metrics work when no budget is set""" - datadog_llm_obs_logger = DataDogLLMObsLogger() - - # Create a standard logging payload without budget metadata - payload = create_standard_logging_payload_with_spend_metrics() - - # Remove budget-related metadata to test no-budget scenario - payload["metadata"].pop("user_api_key_max_budget", None) - payload["metadata"].pop("user_api_key_budget_reset_at", None) - - # Test the _get_spend_metrics method - spend_metrics = datadog_llm_obs_logger._get_spend_metrics(payload) - - # Verify only response cost is present - assert "response_cost" in spend_metrics - assert spend_metrics["response_cost"] == 0.15 - - # Budget metrics should not be present - assert "user_api_key_max_budget" not in spend_metrics - assert "user_api_key_budget_reset_at" not in spend_metrics - - print(f"Spend metrics (no budget): {spend_metrics}") - - -@pytest.mark.asyncio -async def test_spend_metrics_in_datadog_payload(mock_env_vars): - """Test that spend metrics are correctly included in DataDog LLM Observability payloads""" - from datetime import datetime - - datadog_llm_obs_logger = DataDogLLMObsLogger() - - standard_payload = create_standard_logging_payload_with_spend_metrics() - - kwargs = { - "standard_logging_object": standard_payload, - "litellm_params": {"metadata": {}}, - } - - start_time = datetime.now() - end_time = datetime.now() - - payload = datadog_llm_obs_logger.create_llm_obs_payload( - kwargs, start_time, end_time - ) - - # Verify basic payload structure - assert payload.get("name") == "litellm_llm_call" - assert payload.get("status") == "ok" - - # Verify spend metrics are included in metadata - meta = payload.get("meta", {}) - assert meta is not None, "Meta section should exist in payload" - - metadata = meta.get("metadata", {}) - assert metadata is not None, "Metadata section should exist in meta" - - spend_metrics = metadata.get("spend_metrics", {}) - assert spend_metrics, "Spend metrics should exist in metadata" - - # Check that all metrics are present - assert "response_cost" in spend_metrics - assert "user_api_key_spend" in spend_metrics - assert "user_api_key_max_budget" in spend_metrics - assert "user_api_key_budget_reset_at" in spend_metrics - - # Verify the values are correct - assert spend_metrics["response_cost"] == 0.15 # response_cost - assert spend_metrics["user_api_key_spend"] == 0.67 # lol - assert spend_metrics["user_api_key_max_budget"] == 10.0 # max budget - - # Verify budget reset is a datetime string in ISO format - budget_reset = spend_metrics["user_api_key_budget_reset_at"] - assert isinstance(budget_reset, str) - print( - f"Budget reset in payload: {budget_reset}" - ) # In StandardLoggingUserAPIKeyMetadata - user_api_key_budget_reset_at: Optional[str] = None - - # In DDLLMObsSpendMetrics - user_api_key_budget_reset_at: str - # Should be close to 10 days from now - from datetime import datetime, timezone - - budget_reset_dt = datetime.fromisoformat(budget_reset.replace("Z", "+00:00")) - now = datetime.now(timezone.utc) - time_diff = (budget_reset_dt - now).total_seconds() / 86400 # days - assert 9.5 <= time_diff <= 10.5 # Should be close to 10 days diff --git a/tests/test_litellm/llms/pass_through/guardrail_translation/test_handler.py b/tests/test_litellm/llms/pass_through/guardrail_translation/test_handler.py deleted file mode 100644 index 1043c26c6ec..00000000000 --- a/tests/test_litellm/llms/pass_through/guardrail_translation/test_handler.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Tests for the guardrail_translation_mappings registry. - -Validates: -- allm_passthrough_route is registered in the mappings (regression: this was the bug) -""" - -from litellm.llms.pass_through.guardrail_translation import ( - guardrail_translation_mappings, -) -from litellm.llms.pass_through.guardrail_translation.handler import ( - LlmPassthroughRouteHandler, -) -from litellm.types.utils import CallTypes - - -class TestRegistry: - def test_allm_passthrough_route_registered(self): - """Regression: missing this mapping was the root cause of the bug.""" - assert CallTypes.allm_passthrough_route in guardrail_translation_mappings - - def test_allm_passthrough_route_maps_to_llm_passthrough_route_handler(self): - assert ( - guardrail_translation_mappings[CallTypes.allm_passthrough_route] - is LlmPassthroughRouteHandler - ) - - def test_pass_through_still_registered(self): - from litellm.llms.pass_through.guardrail_translation.handler import ( - PassThroughEndpointHandler, - ) - - assert ( - guardrail_translation_mappings[CallTypes.pass_through] - is PassThroughEndpointHandler - ) - diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py deleted file mode 100644 index 020b5de0a2a..00000000000 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py +++ /dev/null @@ -1,296 +0,0 @@ -""" -Test reasoning content preservation in Responses API transformation -""" - -from unittest.mock import AsyncMock - -from litellm.responses.litellm_completion_transformation.streaming_iterator import ( - LiteLLMCompletionStreamingIterator, -) -from litellm.responses.litellm_completion_transformation.transformation import ( - LiteLLMCompletionResponsesConfig, -) -from litellm.types.utils import ( - Choices, - Delta, - Message, - ModelResponse, - ModelResponseStream, - StreamingChoices, -) - - -class TestReasoningContentStreaming: - """Test reasoning content preservation during streaming""" - - def test_reasoning_content_in_delta(self): - """Test that reasoning content is preserved in streaming deltas""" - # Setup - chunk = ModelResponseStream( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role="assistant", - reasoning_content="Let me think about this problem...", - ), - ) - ], - ) - - mock_stream = AsyncMock() - - iterator = LiteLLMCompletionStreamingIterator( - model="test-model", - litellm_custom_stream_wrapper=mock_stream, - request_input="Test input", - responses_api_request={}, - ) - - # Execute - transformed_chunk = ( - iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) - ) - - # Assert - assert transformed_chunk.delta == "Let me think about this problem..." - assert transformed_chunk.type == "response.reasoning_summary_text.delta" - - def test_mixed_content_and_reasoning(self): - """Test handling of both content and reasoning content""" - # Setup - chunk = ModelResponseStream( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="Here is the answer", - role="assistant", - reasoning_content="First, let me analyze...", - ), - ) - ], - ) - - mock_stream = AsyncMock() - iterator = LiteLLMCompletionStreamingIterator( - model="test-model", - litellm_custom_stream_wrapper=mock_stream, - request_input="Test input", - responses_api_request={}, - ) - - # Execute - transformed_chunk = ( - iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) - ) - - # Assert - assert transformed_chunk.delta == "First, let me analyze..." - assert transformed_chunk.type == "response.reasoning_summary_text.delta" - - def test_no_reasoning_content(self): - """Test handling when no reasoning content is present""" - # Setup - chunk = ModelResponseStream( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="Regular content only", - role="assistant", - ), - ) - ], - ) - - mock_stream = AsyncMock() - iterator = LiteLLMCompletionStreamingIterator( - model="test-model", - litellm_custom_stream_wrapper=mock_stream, - request_input="Test input", - responses_api_request={}, - ) - - # Execute - transformed_chunk = ( - iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) - ) - - # Assert - assert transformed_chunk.delta == "Regular content only" - assert transformed_chunk.type == "response.output_text.delta" - - -class TestReasoningContentFinalResponse: - """Test reasoning content preservation in final response transformation""" - - def test_reasoning_content_in_final_response(self): - """Test that reasoning content is included in final response""" - # Setup - response = ModelResponse( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="Here is my answer", - role="assistant", - reasoning_content="Let me think step by step about this problem...", - ), - ) - ], - ) - - # Execute - responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="Test input", - responses_api_request={}, - chat_completion_response=response, - ) - - # Assert - assert hasattr(responses_api_response, "output") - assert len(responses_api_response.output) > 0 - - reasoning_items = [ - item for item in responses_api_response.output if item.type == "reasoning" - ] - assert len(reasoning_items) > 0, "No reasoning item found in output" - - reasoning_item = reasoning_items[0] - assert ( - reasoning_item.content[0].text - == "Let me think step by step about this problem..." - ) - - def test_no_reasoning_content_in_response(self): - """Test handling when no reasoning content in response""" - # Setup - response = ModelResponse( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="Simple answer", - role="assistant", - ), - ) - ], - ) - - # Execute - responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="Test input", - responses_api_request={}, - chat_completion_response=response, - ) - - # Assert - reasoning_items = [ - item for item in responses_api_response.output if item.type == "reasoning" - ] - assert ( - len(reasoning_items) == 0 - ), "Should have no reasoning items when no reasoning content present" - - def test_multiple_choices_with_reasoning(self): - """Test handling multiple choices, first with reasoning content""" - # Setup - response = ModelResponse( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="First answer", - role="assistant", - reasoning_content="Reasoning for first answer", - ), - ), - Choices( - finish_reason="stop", - index=1, - message=Message( - content="Second answer", - role="assistant", - reasoning_content="Reasoning for second answer", - ), - ), - ], - ) - - # Execute - responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="Test input", - responses_api_request={}, - chat_completion_response=response, - ) - - # Assert - reasoning_items = [ - item for item in responses_api_response.output if item.type == "reasoning" - ] - assert len(reasoning_items) == 1, "Should have exactly one reasoning item" - assert reasoning_items[0].content[0].text == "Reasoning for first answer" - - -def test_streaming_chunk_id_raw(): - """Test that streaming chunk IDs are raw (not encoded) to match OpenAI format""" - chunk = ModelResponseStream( - id="chunk-123", - created=1234567890, - model="test-model", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta(content="Hello", role="assistant"), - ) - ], - ) - - iterator = LiteLLMCompletionStreamingIterator( - model="test-model", - litellm_custom_stream_wrapper=AsyncMock(), - request_input="Test input", - responses_api_request={}, - custom_llm_provider="openai", - litellm_metadata={"model_info": {"id": "gpt-4"}}, - ) - - result = iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) - - # Streaming chunk IDs should be raw (like OpenAI's msg_xxx format) - assert result.item_id == "chunk-123" # Should be raw, not encoded - assert not result.item_id.startswith("resp_") # Should NOT have resp_ prefix diff --git a/tests/test_litellm/test_azure_video_router.py b/tests/test_litellm/test_azure_video_router.py deleted file mode 100644 index e7e2e0a01ea..00000000000 --- a/tests/test_litellm/test_azure_video_router.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Test suite for Azure video router functionality. -Tests that the router method gets called correctly for Azure video generation. -""" - -import pytest -from unittest.mock import Mock, patch, MagicMock -import litellm - - -class TestAzureVideoRouter: - """Test suite for Azure video router functionality""" - - def setup_method(self): - """Setup test fixtures""" - self.model = "azure/sora-2" - self.prompt = "A beautiful sunset over mountains" - self.seconds = "5" - self.size = "1280x720" - - @patch("litellm.videos.main.base_llm_http_handler") - def test_azure_video_generation_router_call_mock(self, mock_handler): - """Test that Azure video generation calls the router method with mock response""" - # Setup mock response - mock_response = { - "id": "video_123", - "model": "sora-2", - "object": "video", - "status": "processing", - "created_at": 1234567890, - "progress": 0, - } - - # Configure the mock handler - mock_handler.video_generation_handler.return_value = mock_response - - # Call the video generation function with mock response - result = litellm.video_generation( - prompt=self.prompt, - model=self.model, - seconds=self.seconds, - size=self.size, - custom_llm_provider="azure", - mock_response=mock_response, - ) - - # Verify the result is a VideoObject with the expected data - assert result.id == mock_response["id"] - assert result.model == mock_response["model"] - assert result.object == mock_response["object"] - assert result.status == mock_response["status"] - assert result.created_at == mock_response["created_at"] - assert result.progress == mock_response["progress"] From e1afe2e29cee700710faa85063a9a0f7927104f6 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 16:42:15 -0700 Subject: [PATCH 007/672] test(e2e): bound the post-/model/new servable wait at 40s _await_model_servable used poll_timeout (120s), the spend/log read-back budget. A stuck model reload therefore stalled every suite that creates a deployment for two minutes before failing Give create_model a fixed harness middle ground: model_servable_timeout=40s, polled every 2s, with each /v1/models call capped at 5s and clamped to the remaining deadline so one slow GET cannot overrun the wait. Happy path still returns on the first listing. Not derived from proxy general_settings or env Transport.get accepts an optional per-call timeout for that clamp. Unit tests cover the deadline arithmetic and clamp without a live proxy (cherry picked from commit c082a0e6488f50978bf5255f6b5298ba7e8fd8da) --- tests/e2e/proxy_client.py | 134 ++++++++++-- tests/e2e/test_proxy_client_model_servable.py | 190 ++++++++++++++++++ tests/e2e/transport.py | 13 +- 3 files changed, 313 insertions(+), 24 deletions(-) create mode 100644 tests/e2e/test_proxy_client_model_servable.py diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 6c6b948e29c..87693175e62 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -75,12 +75,95 @@ from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] +# After /model/new, poll the data plane until the model is listed (or fail). +# Shorter than poll_timeout (spend/log read-backs ~120s); longer than a single +# request. 40s is the harness middle ground: happy path returns on the first +# poll, a stuck reload fails in under a minute instead of two. +MODEL_SERVABLE_TIMEOUT = 40.0 +MODEL_SERVABLE_INTERVAL = 2.0 +# Cap each /v1/models poll so one slow request cannot outlast the budget. +# Clamped further to remaining deadline inside await_servable. +MODEL_SERVABLE_REQUEST_TIMEOUT = 5.0 + + +@dataclass(frozen=True, slots=True) +class Servable: + """The data plane listed the model within the deadline.""" + + +@dataclass(frozen=True, slots=True) +class NotServable: + """The deadline passed without the data plane listing the model. + + `last_result` is the final /v1/models read, so the caller can tell "the proxy + answered but omitted the model" (propagation) from "the read itself failed" + (network/auth) when reporting.""" + + last_result: Result[ModelsListResponse] | None + + +ServableOutcome = Servable | NotServable + + +def await_servable( + list_models: Callable[[float], Result[ModelsListResponse]], + *, + model_name: str, + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> ServableOutcome: + """Poll `list_models` until the data plane lists `model_name` or `timeout` elapses. + + `list_models` receives the per-poll request timeout, clamped to the remaining + deadline so a slow final poll cannot overrun the overall budget. Clock and sleep + are injected so this is exercised without wall-clock waits. Always polls at least + once when the loop starts with a positive budget.""" + deadline = now() + timeout + last_result: Result[ModelsListResponse] | None = None + while True: + remaining = deadline - now() + if remaining <= 0 and last_result is not None: + return NotServable(last_result=last_result) + poll_timeout = min(request_timeout, remaining) if remaining > 0 else request_timeout + last_result = list_models(poll_timeout) + if isinstance(last_result, Success) and any( + entry.id == model_name for entry in last_result.data.data + ): + return Servable() + if now() + interval >= deadline: + return NotServable(last_result=last_result) + sleep(interval) + + +def servable_timeout_message( + *, + model_name: str, + timeout: float, + last_result: Result[ModelsListResponse] | None, +) -> str: + last_error = ( + f"; last /v1/models poll did not succeed: {last_result}" + if last_result is not None and not isinstance(last_result, Success) + else "" + ) + return ( + f"model {model_name!r} was created but never became servable on the data " + f"plane within {timeout}s of /model/new (control/data-plane propagation or " + f"STORE_MODEL_IN_DB reload issue){last_error}" + ) + @dataclass(frozen=True, slots=True) class ProxyClient: transport: Transport poll_timeout: float = 120.0 poll_interval: float = 5.0 + model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT + model_servable_interval: float = MODEL_SERVABLE_INTERVAL + model_servable_request_timeout: float = MODEL_SERVABLE_REQUEST_TIMEOUT # ---- keys / customers (satisfies lifecycle.ResourceClient) ---------- @@ -167,7 +250,12 @@ class ProxyClient: this returns can race the reload and 400 with "Invalid model name passed". We therefore poll the data-plane /v1/models until the model appears before handing back, so callers can invoke it immediately. In the monolithic case - it is already present on the first poll, so this adds one request.""" + it is already present on the first poll, so this adds one request. + + The wait is bounded by `model_servable_timeout` rather than the much longer + `poll_timeout` used for batched read-backs, so a stuck reload fails in under + a minute instead of two. Happy path still returns as soon as /v1/models lists + the model (usually the first poll).""" model_id = unwrap( self.transport.post( "/model/new", @@ -185,32 +273,34 @@ class ProxyClient: def _await_model_servable(self, model_name: str) -> None: """Block until the data plane lists `model_name`, or fail loudly if it does - not within poll_timeout (a real propagation/config problem, surfaced here - instead of as a downstream "Invalid model name passed").""" - deadline = time.monotonic() + self.poll_timeout - last_result: Result[ModelsListResponse] | None = None - while time.monotonic() < deadline: - last_result = self.transport.get( + not within model_servable_timeout (a real propagation/config problem, + surfaced here instead of as a downstream "Invalid model name passed").""" + outcome = await_servable( + lambda poll_timeout: self.transport.get( "/v1/models", headers=self.transport.master, params=NoBody(), response_type=ModelsListResponse, - ) - if isinstance(last_result, Success) and any( - entry.id == model_name for entry in last_result.data.data - ): + timeout=poll_timeout, + ), + model_name=model_name, + timeout=self.model_servable_timeout, + interval=self.model_servable_interval, + request_timeout=self.model_servable_request_timeout, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case Servable(): return - time.sleep(self.poll_interval) - last_error = ( - f"; last /v1/models poll did not succeed: {last_result}" - if last_result is not None and not isinstance(last_result, Success) - else "" - ) - raise AssertionError( - f"model {model_name!r} was created but never became servable on the data " - f"plane within {self.poll_timeout}s of /model/new (control/data-plane " - f"propagation or STORE_MODEL_IN_DB reload issue){last_error}" - ) + case NotServable(last_result=last_result): + raise AssertionError( + servable_timeout_message( + model_name=model_name, + timeout=self.model_servable_timeout, + last_result=last_result, + ) + ) def update_model(self, model_id: str, litellm_params: LiteLLMParamsBody) -> None: """Merge `litellm_params` over the deployment `model_id`'s stored params via diff --git a/tests/e2e/test_proxy_client_model_servable.py b/tests/e2e/test_proxy_client_model_servable.py new file mode 100644 index 00000000000..0cc63f882e2 --- /dev/null +++ b/tests/e2e/test_proxy_client_model_servable.py @@ -0,0 +1,190 @@ +"""Harness coverage for the bounded wait after /model/new (no live proxy). + +Model propagation is polled to a deadline so a stuck control/data-plane reload fails +fast instead of stalling every test that creates a model. The clock and sleep are +injected, so these assert the deadline arithmetic without waiting. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from e2e_http import NetworkError, Result, Success +from models import ModelListEntry, ModelsListResponse +from proxy_client import ( + MODEL_SERVABLE_REQUEST_TIMEOUT, + MODEL_SERVABLE_TIMEOUT, + NotServable, + Servable, + await_servable, + servable_timeout_message, +) + + +def _listing(*model_names: str) -> Result[ModelsListResponse]: + return Success( + status_code=200, + data=ModelsListResponse(data=tuple(ModelListEntry(id=name) for name in model_names)), + ) + + +@dataclass(slots=True) +class FakeClock: + """A clock that only advances when the code under test sleeps or a slow poll runs.""" + + seconds: float = 0.0 + slept: list[float] = field(default_factory=list) # mutable-ok: records calls for assertions + + def now(self) -> float: + return self.seconds + + def sleep(self, duration: float) -> None: + self.slept.append(duration) + self.seconds += duration + + +@dataclass(slots=True) +class FakeModelList: + """Returns each queued /v1/models read in turn, repeating the last forever.""" + + responses: tuple[Result[ModelsListResponse], ...] + calls: int = 0 + timeouts: list[float] = field(default_factory=list) # mutable-ok: records call timeouts + + def __call__(self, request_timeout: float) -> Result[ModelsListResponse]: + self.timeouts.append(request_timeout) + response = self.responses[min(self.calls, len(self.responses) - 1)] + self.calls += 1 + return response + + +def test_returns_servable_on_first_listing_without_sleeping() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("my-model"),)) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert list_models.calls == 1 + assert list_models.timeouts == [5.0] + assert clock.slept == [] + + +def test_polls_until_the_model_appears() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("other"), _listing("other"), _listing("other", "my-model"))) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert list_models.calls == 3 + assert clock.seconds == 4.0 + + +def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("other"),)) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=10.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert clock.seconds == 8.0 + assert list_models.calls == 5 + assert list_models.timeouts == [5.0, 5.0, 5.0, 4.0, 2.0] + + +def test_does_not_wait_past_the_overall_budget() -> None: + clock = FakeClock() + + outcome = await_servable( + FakeModelList(responses=(_listing("other"),)), + model_name="my-model", + timeout=MODEL_SERVABLE_TIMEOUT, + interval=2.0, + request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert clock.seconds <= MODEL_SERVABLE_TIMEOUT + + +def test_clamps_request_timeout_to_remaining_deadline() -> None: + """A slow final poll must not receive the full request cap when less budget remains. + + Without the clamp, remaining=3 and cap=5 lets the transport block for 5s and the + overall wait overruns model_servable_timeout by up to ~cap seconds. + """ + clock = FakeClock() + timeouts: list[float] = [] + + def list_models(request_timeout: float) -> Result[ModelsListResponse]: + timeouts.append(request_timeout) + clock.seconds += request_timeout + return _listing("other") + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=10.0, + interval=2.0, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert timeouts[0] == 5.0 + assert any(timeout < 5.0 for timeout in timeouts) + assert timeouts[-1] == 3.0 + assert clock.seconds <= 10.0 + + +def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: + clock = FakeClock() + unreachable: Result[ModelsListResponse] = NetworkError(message="connection refused") + + outcome = await_servable( + FakeModelList(responses=(unreachable,)), + model_name="my-model", + timeout=1.0, + interval=0.5, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == NotServable(last_result=unreachable) + message = servable_timeout_message(model_name="my-model", timeout=1.0, last_result=unreachable) + assert "connection refused" in message + + listed_without_model = _listing("other") + propagation_message = servable_timeout_message( + model_name="my-model", timeout=1.0, last_result=listed_without_model + ) + assert "did not succeed" not in propagation_message diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index a6adf83ed1f..27b11befc8e 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -58,6 +58,7 @@ class Transport(Protocol): headers: BaseModel, params: BaseModel, response_type: type[R], + timeout: float | None = None, ) -> Result[R]: ... def delete[R: BaseModel]( @@ -136,13 +137,16 @@ class HttpTransport: headers: BaseModel, params: BaseModel, response_type: type[R], + timeout: float | None = None, ) -> Result[R]: + """`timeout` overrides the transport-wide request_timeout for this call, for + pollers whose own deadline is shorter than it.""" return e2e_http.get( self._url(path), headers=headers, params=params, response_type=response_type, - timeout=self.request_timeout, + timeout=self.request_timeout if timeout is None else timeout, ) def delete[R: BaseModel]( @@ -336,9 +340,14 @@ class SplitTransport: headers: BaseModel, params: BaseModel, response_type: type[R], + timeout: float | None = None, ) -> Result[R]: return self._route(path).get( - path, headers=headers, params=params, response_type=response_type + path, + headers=headers, + params=params, + response_type=response_type, + timeout=timeout, ) def delete[R: BaseModel]( From 5aa66ea33e6c194f66d360e22292f22eddac38e7 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:09:20 -0700 Subject: [PATCH 008/672] fix(e2e): wait one default DB reload interval of continuous listing create_model returned after the first /v1/models hit that listed the model, so chat could still land on a cold gateway worker (numWorkers>1 / peer pod) and 400 Invalid model name. Require continuous listing for the product default add_deployment interval (30s) after first sight so every worker has synced from the DB; first listing still bounded at 40s (cherry picked from commit 7d1ee2ff861b970f6de3f6759ff015947af9d2a1) --- tests/e2e/proxy_client.py | 85 ++++++++---- tests/e2e/test_proxy_client_model_servable.py | 125 ++++++++++++------ 2 files changed, 148 insertions(+), 62 deletions(-) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 87693175e62..dfbb90ac08e 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -75,14 +75,17 @@ from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] -# After /model/new, poll the data plane until the model is listed (or fail). -# Shorter than poll_timeout (spend/log read-backs ~120s); longer than a single -# request. 40s is the harness middle ground: happy path returns on the first -# poll, a stuck reload fails in under a minute instead of two. +# After /model/new, the control-plane writer reloads itself immediately, but every +# other gateway worker (and peer pod) only picks the model up on its add_deployment +# job. That job runs every proxy_config_reload_interval_seconds (product default 30). +# A single /v1/models hit can land on a hot worker while the next /chat hits a cold +# one ("Invalid model name"). Wait for first listing within MODEL_SERVABLE_TIMEOUT, +# then require continuous listing for MODEL_SERVABLE_DB_SYNC_SECONDS (the default +# reload interval) so every worker has had a chance to sync from the DB. MODEL_SERVABLE_TIMEOUT = 40.0 +MODEL_SERVABLE_DB_SYNC_SECONDS = 30.0 MODEL_SERVABLE_INTERVAL = 2.0 -# Cap each /v1/models poll so one slow request cannot outlast the budget. -# Clamped further to remaining deadline inside await_servable. +# Cap each /v1/models poll so one slow request cannot outlast the remaining budget. MODEL_SERVABLE_REQUEST_TIMEOUT = 5.0 @@ -112,29 +115,55 @@ def await_servable( timeout: float, interval: float, request_timeout: float, + db_sync_seconds: float, now: Callable[[], float], sleep: Callable[[float], None], ) -> ServableOutcome: - """Poll `list_models` until the data plane lists `model_name` or `timeout` elapses. + """Poll until `model_name` is listed long enough for every worker to DB-sync. - `list_models` receives the per-poll request timeout, clamped to the remaining - deadline so a slow final poll cannot overrun the overall budget. Clock and sleep - are injected so this is exercised without wall-clock waits. Always polls at least - once when the loop starts with a positive budget.""" - deadline = now() + timeout + First listing must happen within `timeout`. After that, the model must stay + listed continuously for `db_sync_seconds` (any miss resets the continuous + window). `db_sync_seconds=0` returns on the first listing. Each poll's request + timeout is clamped to the remaining budget. Clock and sleep are injected.""" + started = now() + first_seen_at: float | None = None last_result: Result[ModelsListResponse] | None = None while True: - remaining = deadline - now() + t = now() + if first_seen_at is None: + deadline = started + timeout + else: + deadline = first_seen_at + db_sync_seconds + remaining = deadline - t if remaining <= 0 and last_result is not None: + if first_seen_at is not None and db_sync_seconds <= 0: + return Servable() + if first_seen_at is not None and t - first_seen_at >= db_sync_seconds: + return Servable() return NotServable(last_result=last_result) poll_timeout = min(request_timeout, remaining) if remaining > 0 else request_timeout last_result = list_models(poll_timeout) - if isinstance(last_result, Success) and any( + listed = isinstance(last_result, Success) and any( entry.id == model_name for entry in last_result.data.data - ): + ) + t = now() + if not listed: + first_seen_at = None + elif first_seen_at is None: + first_seen_at = t + if db_sync_seconds <= 0: + return Servable() + elif t - first_seen_at >= db_sync_seconds: return Servable() - if now() + interval >= deadline: - return NotServable(last_result=last_result) + if first_seen_at is None: + if now() + interval >= started + timeout: + return NotServable(last_result=last_result) + elif now() + interval >= first_seen_at + db_sync_seconds: + # Final stretch: sleep only the remainder of the continuous window. + remainder = first_seen_at + db_sync_seconds - now() + if remainder > 0: + sleep(remainder) + continue sleep(interval) @@ -142,6 +171,7 @@ def servable_timeout_message( *, model_name: str, timeout: float, + db_sync_seconds: float, last_result: Result[ModelsListResponse] | None, ) -> str: last_error = ( @@ -151,7 +181,8 @@ def servable_timeout_message( ) return ( f"model {model_name!r} was created but never became servable on the data " - f"plane within {timeout}s of /model/new (control/data-plane propagation or " + f"plane within {timeout}s of first listing (plus {db_sync_seconds}s continuous " + f"DB sync) after /model/new (control/data-plane propagation or " f"STORE_MODEL_IN_DB reload issue){last_error}" ) @@ -162,6 +193,7 @@ class ProxyClient: poll_timeout: float = 120.0 poll_interval: float = 5.0 model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT + model_servable_db_sync_seconds: float = MODEL_SERVABLE_DB_SYNC_SECONDS model_servable_interval: float = MODEL_SERVABLE_INTERVAL model_servable_request_timeout: float = MODEL_SERVABLE_REQUEST_TIMEOUT @@ -252,10 +284,10 @@ class ProxyClient: handing back, so callers can invoke it immediately. In the monolithic case it is already present on the first poll, so this adds one request. - The wait is bounded by `model_servable_timeout` rather than the much longer - `poll_timeout` used for batched read-backs, so a stuck reload fails in under - a minute instead of two. Happy path still returns as soon as /v1/models lists - the model (usually the first poll).""" + First listing must arrive within `model_servable_timeout` (not the longer + spend `poll_timeout`). The model must then stay listed for + `model_servable_db_sync_seconds` (product default DB reload interval) so every + gateway worker has run add_deployment before callers use the model.""" model_id = unwrap( self.transport.post( "/model/new", @@ -272,9 +304,10 @@ class ProxyClient: return model_id def _await_model_servable(self, model_name: str) -> None: - """Block until the data plane lists `model_name`, or fail loudly if it does - not within model_servable_timeout (a real propagation/config problem, - surfaced here instead of as a downstream "Invalid model name passed").""" + """Block until the data plane lists `model_name` long enough for DB sync. + + Fails if first listing misses model_servable_timeout, or if continuous listing + for model_servable_db_sync_seconds never holds (multi-worker / peer reload).""" outcome = await_servable( lambda poll_timeout: self.transport.get( "/v1/models", @@ -287,6 +320,7 @@ class ProxyClient: timeout=self.model_servable_timeout, interval=self.model_servable_interval, request_timeout=self.model_servable_request_timeout, + db_sync_seconds=self.model_servable_db_sync_seconds, now=time.monotonic, sleep=time.sleep, ) @@ -298,6 +332,7 @@ class ProxyClient: servable_timeout_message( model_name=model_name, timeout=self.model_servable_timeout, + db_sync_seconds=self.model_servable_db_sync_seconds, last_result=last_result, ) ) diff --git a/tests/e2e/test_proxy_client_model_servable.py b/tests/e2e/test_proxy_client_model_servable.py index 0cc63f882e2..c9edd148c2b 100644 --- a/tests/e2e/test_proxy_client_model_servable.py +++ b/tests/e2e/test_proxy_client_model_servable.py @@ -1,8 +1,9 @@ """Harness coverage for the bounded wait after /model/new (no live proxy). -Model propagation is polled to a deadline so a stuck control/data-plane reload fails -fast instead of stalling every test that creates a model. The clock and sleep are -injected, so these assert the deadline arithmetic without waiting. +create_model must wait for the product default DB reload interval of continuous +listing so multi-worker gateways finish add_deployment before callers use the +model. Clock and sleep are injected so these assert the deadline arithmetic +without wall-clock waits. """ from __future__ import annotations @@ -12,6 +13,7 @@ from dataclasses import dataclass, field from e2e_http import NetworkError, Result, Success from models import ModelListEntry, ModelsListResponse from proxy_client import ( + MODEL_SERVABLE_DB_SYNC_SECONDS, MODEL_SERVABLE_REQUEST_TIMEOUT, MODEL_SERVABLE_TIMEOUT, NotServable, @@ -30,8 +32,6 @@ def _listing(*model_names: str) -> Result[ModelsListResponse]: @dataclass(slots=True) class FakeClock: - """A clock that only advances when the code under test sleeps or a slow poll runs.""" - seconds: float = 0.0 slept: list[float] = field(default_factory=list) # mutable-ok: records calls for assertions @@ -45,8 +45,6 @@ class FakeClock: @dataclass(slots=True) class FakeModelList: - """Returns each queued /v1/models read in turn, repeating the last forever.""" - responses: tuple[Result[ModelsListResponse], ...] calls: int = 0 timeouts: list[float] = field(default_factory=list) # mutable-ok: records call timeouts @@ -58,7 +56,7 @@ class FakeModelList: return response -def test_returns_servable_on_first_listing_without_sleeping() -> None: +def test_returns_on_first_listing_when_db_sync_is_zero() -> None: clock = FakeClock() list_models = FakeModelList(responses=(_listing("my-model"),)) @@ -68,17 +66,64 @@ def test_returns_servable_on_first_listing_without_sleeping() -> None: timeout=40.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=0.0, now=clock.now, sleep=clock.sleep, ) assert outcome == Servable() assert list_models.calls == 1 - assert list_models.timeouts == [5.0] assert clock.slept == [] -def test_polls_until_the_model_appears() -> None: +def test_requires_continuous_listing_for_default_db_sync_interval() -> None: + clock = FakeClock() + list_models = FakeModelList(responses=(_listing("my-model"),)) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert clock.seconds >= MODEL_SERVABLE_DB_SYNC_SECONDS + assert list_models.calls >= 2 + + +def test_resets_db_sync_window_when_a_poll_misses() -> None: + clock = FakeClock() + list_models = FakeModelList( + responses=( + _listing("my-model"), + _listing("my-model"), + _listing("other"), + _listing("my-model"), + ) + ) + + outcome = await_servable( + list_models, + model_name="my-model", + timeout=40.0, + interval=2.0, + request_timeout=5.0, + db_sync_seconds=6.0, + now=clock.now, + sleep=clock.sleep, + ) + + assert outcome == Servable() + assert list_models.calls >= 4 + assert clock.seconds >= 6.0 + + +def test_polls_until_the_model_first_appears() -> None: clock = FakeClock() list_models = FakeModelList(responses=(_listing("other"), _listing("other"), _listing("other", "my-model"))) @@ -88,6 +133,7 @@ def test_polls_until_the_model_appears() -> None: timeout=40.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=0.0, now=clock.now, sleep=clock.sleep, ) @@ -97,7 +143,7 @@ def test_polls_until_the_model_appears() -> None: assert clock.seconds == 4.0 -def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: +def test_gives_up_if_first_listing_never_arrives() -> None: clock = FakeClock() list_models = FakeModelList(responses=(_listing("other"),)) @@ -107,6 +153,7 @@ def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: timeout=10.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=30.0, now=clock.now, sleep=clock.sleep, ) @@ -114,32 +161,9 @@ def test_gives_up_at_the_deadline_rather_than_polling_forever() -> None: assert isinstance(outcome, NotServable) assert clock.seconds == 8.0 assert list_models.calls == 5 - assert list_models.timeouts == [5.0, 5.0, 5.0, 4.0, 2.0] - - -def test_does_not_wait_past_the_overall_budget() -> None: - clock = FakeClock() - - outcome = await_servable( - FakeModelList(responses=(_listing("other"),)), - model_name="my-model", - timeout=MODEL_SERVABLE_TIMEOUT, - interval=2.0, - request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert clock.seconds <= MODEL_SERVABLE_TIMEOUT def test_clamps_request_timeout_to_remaining_deadline() -> None: - """A slow final poll must not receive the full request cap when less budget remains. - - Without the clamp, remaining=3 and cap=5 lets the transport block for 5s and the - overall wait overruns model_servable_timeout by up to ~cap seconds. - """ clock = FakeClock() timeouts: list[float] = [] @@ -154,6 +178,7 @@ def test_clamps_request_timeout_to_remaining_deadline() -> None: timeout=10.0, interval=2.0, request_timeout=5.0, + db_sync_seconds=30.0, now=clock.now, sleep=clock.sleep, ) @@ -161,10 +186,27 @@ def test_clamps_request_timeout_to_remaining_deadline() -> None: assert isinstance(outcome, NotServable) assert timeouts[0] == 5.0 assert any(timeout < 5.0 for timeout in timeouts) - assert timeouts[-1] == 3.0 assert clock.seconds <= 10.0 +def test_does_not_wait_past_first_listing_budget_when_missing() -> None: + clock = FakeClock() + + outcome = await_servable( + FakeModelList(responses=(_listing("other"),)), + model_name="my-model", + timeout=MODEL_SERVABLE_TIMEOUT, + interval=2.0, + request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, + db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, + now=clock.now, + sleep=clock.sleep, + ) + + assert isinstance(outcome, NotServable) + assert clock.seconds <= MODEL_SERVABLE_TIMEOUT + + def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: clock = FakeClock() unreachable: Result[ModelsListResponse] = NetworkError(message="connection refused") @@ -175,16 +217,25 @@ def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: timeout=1.0, interval=0.5, request_timeout=5.0, + db_sync_seconds=0.0, now=clock.now, sleep=clock.sleep, ) assert outcome == NotServable(last_result=unreachable) - message = servable_timeout_message(model_name="my-model", timeout=1.0, last_result=unreachable) + message = servable_timeout_message( + model_name="my-model", + timeout=1.0, + db_sync_seconds=0.0, + last_result=unreachable, + ) assert "connection refused" in message listed_without_model = _listing("other") propagation_message = servable_timeout_message( - model_name="my-model", timeout=1.0, last_result=listed_without_model + model_name="my-model", + timeout=1.0, + db_sync_seconds=30.0, + last_result=listed_without_model, ) assert "did not succeed" not in propagation_message From 5953a66eab8212beac24d3265cd118b7f4a51176 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:10:04 -0700 Subject: [PATCH 009/672] test(e2e): drop proxy_client model-servable unit tests Keep the create_model DB-sync wait in the harness; the pure-function unit file is not needed for this PR (cherry picked from commit 89204651d1a4537c6f21550c5ab85448ae0923f8) --- tests/e2e/test_proxy_client_model_servable.py | 241 ------------------ 1 file changed, 241 deletions(-) delete mode 100644 tests/e2e/test_proxy_client_model_servable.py diff --git a/tests/e2e/test_proxy_client_model_servable.py b/tests/e2e/test_proxy_client_model_servable.py deleted file mode 100644 index c9edd148c2b..00000000000 --- a/tests/e2e/test_proxy_client_model_servable.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Harness coverage for the bounded wait after /model/new (no live proxy). - -create_model must wait for the product default DB reload interval of continuous -listing so multi-worker gateways finish add_deployment before callers use the -model. Clock and sleep are injected so these assert the deadline arithmetic -without wall-clock waits. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from e2e_http import NetworkError, Result, Success -from models import ModelListEntry, ModelsListResponse -from proxy_client import ( - MODEL_SERVABLE_DB_SYNC_SECONDS, - MODEL_SERVABLE_REQUEST_TIMEOUT, - MODEL_SERVABLE_TIMEOUT, - NotServable, - Servable, - await_servable, - servable_timeout_message, -) - - -def _listing(*model_names: str) -> Result[ModelsListResponse]: - return Success( - status_code=200, - data=ModelsListResponse(data=tuple(ModelListEntry(id=name) for name in model_names)), - ) - - -@dataclass(slots=True) -class FakeClock: - seconds: float = 0.0 - slept: list[float] = field(default_factory=list) # mutable-ok: records calls for assertions - - def now(self) -> float: - return self.seconds - - def sleep(self, duration: float) -> None: - self.slept.append(duration) - self.seconds += duration - - -@dataclass(slots=True) -class FakeModelList: - responses: tuple[Result[ModelsListResponse], ...] - calls: int = 0 - timeouts: list[float] = field(default_factory=list) # mutable-ok: records call timeouts - - def __call__(self, request_timeout: float) -> Result[ModelsListResponse]: - self.timeouts.append(request_timeout) - response = self.responses[min(self.calls, len(self.responses) - 1)] - self.calls += 1 - return response - - -def test_returns_on_first_listing_when_db_sync_is_zero() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("my-model"),)) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=0.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert list_models.calls == 1 - assert clock.slept == [] - - -def test_requires_continuous_listing_for_default_db_sync_interval() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("my-model"),)) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert clock.seconds >= MODEL_SERVABLE_DB_SYNC_SECONDS - assert list_models.calls >= 2 - - -def test_resets_db_sync_window_when_a_poll_misses() -> None: - clock = FakeClock() - list_models = FakeModelList( - responses=( - _listing("my-model"), - _listing("my-model"), - _listing("other"), - _listing("my-model"), - ) - ) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=6.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert list_models.calls >= 4 - assert clock.seconds >= 6.0 - - -def test_polls_until_the_model_first_appears() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("other"), _listing("other"), _listing("other", "my-model"))) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=40.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=0.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == Servable() - assert list_models.calls == 3 - assert clock.seconds == 4.0 - - -def test_gives_up_if_first_listing_never_arrives() -> None: - clock = FakeClock() - list_models = FakeModelList(responses=(_listing("other"),)) - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=10.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=30.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert clock.seconds == 8.0 - assert list_models.calls == 5 - - -def test_clamps_request_timeout_to_remaining_deadline() -> None: - clock = FakeClock() - timeouts: list[float] = [] - - def list_models(request_timeout: float) -> Result[ModelsListResponse]: - timeouts.append(request_timeout) - clock.seconds += request_timeout - return _listing("other") - - outcome = await_servable( - list_models, - model_name="my-model", - timeout=10.0, - interval=2.0, - request_timeout=5.0, - db_sync_seconds=30.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert timeouts[0] == 5.0 - assert any(timeout < 5.0 for timeout in timeouts) - assert clock.seconds <= 10.0 - - -def test_does_not_wait_past_first_listing_budget_when_missing() -> None: - clock = FakeClock() - - outcome = await_servable( - FakeModelList(responses=(_listing("other"),)), - model_name="my-model", - timeout=MODEL_SERVABLE_TIMEOUT, - interval=2.0, - request_timeout=MODEL_SERVABLE_REQUEST_TIMEOUT, - db_sync_seconds=MODEL_SERVABLE_DB_SYNC_SECONDS, - now=clock.now, - sleep=clock.sleep, - ) - - assert isinstance(outcome, NotServable) - assert clock.seconds <= MODEL_SERVABLE_TIMEOUT - - -def test_reports_a_failed_read_distinctly_from_a_missing_model() -> None: - clock = FakeClock() - unreachable: Result[ModelsListResponse] = NetworkError(message="connection refused") - - outcome = await_servable( - FakeModelList(responses=(unreachable,)), - model_name="my-model", - timeout=1.0, - interval=0.5, - request_timeout=5.0, - db_sync_seconds=0.0, - now=clock.now, - sleep=clock.sleep, - ) - - assert outcome == NotServable(last_result=unreachable) - message = servable_timeout_message( - model_name="my-model", - timeout=1.0, - db_sync_seconds=0.0, - last_result=unreachable, - ) - assert "connection refused" in message - - listed_without_model = _listing("other") - propagation_message = servable_timeout_message( - model_name="my-model", - timeout=1.0, - db_sync_seconds=30.0, - last_result=listed_without_model, - ) - assert "did not succeed" not in propagation_message From 38d03fd341bf5b7030b2ebbc45c2555a7f1ee1e6 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:16:30 -0700 Subject: [PATCH 010/672] fix(e2e): never skip the final deadline-clamped model-servable poll When less than one full poll interval remained in the first-listing budget, the pre-sleep check returned NotServable without another /v1/models call. Sleep only min(interval, time left) so a model that becomes listable in the last seconds of the timeout still gets a clamped final poll (cherry picked from commit 8439195922c913d118cb146409c75cc081d23e6e) --- tests/e2e/proxy_client.py | 43 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index dfbb90ac08e..36dada5770a 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -124,24 +124,28 @@ def await_servable( First listing must happen within `timeout`. After that, the model must stay listed continuously for `db_sync_seconds` (any miss resets the continuous window). `db_sync_seconds=0` returns on the first listing. Each poll's request - timeout is clamped to the remaining budget. Clock and sleep are injected.""" + timeout is clamped to the remaining budget. Sleeps only min(interval, time left) + so a final deadline-clamped poll is never skipped just because a full interval + does not fit. Clock and sleep are injected.""" started = now() first_seen_at: float | None = None last_result: Result[ModelsListResponse] | None = None while True: t = now() - if first_seen_at is None: - deadline = started + timeout - else: - deadline = first_seen_at + db_sync_seconds - remaining = deadline - t - if remaining <= 0 and last_result is not None: - if first_seen_at is not None and db_sync_seconds <= 0: - return Servable() - if first_seen_at is not None and t - first_seen_at >= db_sync_seconds: + phase_deadline = ( + started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds + ) + remaining = phase_deadline - t + if remaining <= 0: + if ( + last_result is not None + and first_seen_at is not None + and (db_sync_seconds <= 0 or t - first_seen_at >= db_sync_seconds) + ): return Servable() return NotServable(last_result=last_result) - poll_timeout = min(request_timeout, remaining) if remaining > 0 else request_timeout + + poll_timeout = min(request_timeout, remaining) last_result = list_models(poll_timeout) listed = isinstance(last_result, Success) and any( entry.id == model_name for entry in last_result.data.data @@ -155,16 +159,13 @@ def await_servable( return Servable() elif t - first_seen_at >= db_sync_seconds: return Servable() - if first_seen_at is None: - if now() + interval >= started + timeout: - return NotServable(last_result=last_result) - elif now() + interval >= first_seen_at + db_sync_seconds: - # Final stretch: sleep only the remainder of the continuous window. - remainder = first_seen_at + db_sync_seconds - now() - if remainder > 0: - sleep(remainder) - continue - sleep(interval) + + phase_deadline = ( + started + timeout if first_seen_at is None else first_seen_at + db_sync_seconds + ) + wait = min(interval, phase_deadline - now()) + if wait > 0: + sleep(wait) def servable_timeout_message( From 87be33f9354f399100c7ededdfba31b0e831d225 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 17:35:13 -0700 Subject: [PATCH 011/672] fix(e2e): reject first listing that returns after the 40s deadline A poll may start with remaining budget and still return after started+timeout if the transport overruns its clamp. Recheck the first-listing deadline after the response so a late listing does not open the continuous DB-sync phase (cherry picked from commit 7ff2bcbf1498ee82f7dbe0b4330c1ab48927ed01) --- tests/e2e/proxy_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 36dada5770a..b3fc8538322 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -154,6 +154,8 @@ def await_servable( if not listed: first_seen_at = None elif first_seen_at is None: + if t > started + timeout: + return NotServable(last_result=last_result) first_seen_at = t if db_sync_seconds <= 0: return Servable() From 82fa66908bb72748efc574a8a1a8750155a85c14 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 28 Jul 2026 22:15:21 -0700 Subject: [PATCH 012/672] test(e2e): poll MCP tools across multi-worker lag (#35047) * fix(mcp): resolve call_tool by registry without requiring tool map Multi-worker reloads put MCP servers in the registry from the DB but do not re-run tools/list on every process. Gating call_tool on tool_name_to_mcp_server_name_mapping made cold workers 500 with Tool not found after another worker had already listed the tool. Treat a registry match on server id/name/alias as enough; upstream rejects unknown tools * test(e2e): poll MCP register, tools/list, and tools/call across multi-worker lag Stage multi-worker gateways only load MCP servers and tool maps on the process that handled the request. Poll until the server is listed, the tool appears on tools/list, and tools/call is not a cold-worker 500 so key-access and Datadog MCP e2e stop racing the LB * Revert "fix(mcp): resolve call_tool by registry without requiring tool map" This reverts commit 8b56e51e39b876d13d1112efa4130554ddf5f173. * test(e2e): tighten MCP multi-worker lag classifier Only retry tools/call on gateway shapes Tool not found and server_not_found, not any 500 that mentions tool/server not found, so upstream failures are not retried until the poll deadline * test(e2e): drop unit file for MCP lag classifier The live await_call_tool polls already cover multi-worker lag; a separate string-match unit module is not worth keeping (cherry picked from commit c274cf321c5c35c629220a89bb497d15b56f870f) --- tests/e2e/mcp/mcp_client.py | 91 +++++++++++++++++++++- tests/e2e/mcp/test_mcp_access_group_e2e.py | 1 + tests/e2e/mcp/test_mcp_datadog_e2e.py | 28 ++++--- tests/e2e/mcp/test_mcp_key_access_e2e.py | 15 ++-- 4 files changed, 111 insertions(+), 24 deletions(-) diff --git a/tests/e2e/mcp/mcp_client.py b/tests/e2e/mcp/mcp_client.py index 6d0f6ddc760..33ec557c339 100644 --- a/tests/e2e/mcp/mcp_client.py +++ b/tests/e2e/mcp/mcp_client.py @@ -11,13 +11,14 @@ request/response bodies are co-located here because only this suite speaks MCP. from __future__ import annotations +import re import time from collections.abc import Mapping from dataclasses import dataclass from pydantic import BaseModel, ConfigDict, Field, RootModel -from e2e_http import Headers, NoBody, Result, Success, unwrap +from e2e_http import Headers, NoBody, Result, Success, UnknownApiError, unwrap from models import KeyGenerateBody, ObjectPermission from proxy_client import ProxyClient @@ -270,6 +271,60 @@ class McpClient: ) time.sleep(self.proxy.poll_interval) + def await_call_tool( + self, + key: str, + *, + server_id: str, + name: str, + arguments: McpToolArguments, + ) -> McpCallToolResponse: + """Poll tools/call until the result is not a multi-worker registry miss. + + Retries only on the gateway's own cold-worker 500 shapes (Tool + not found / server_not_found). Upstream tool errors and other 500s fail + immediately so non-idempotent calls are not repeated. + """ + deadline = time.monotonic() + self.proxy.poll_timeout + last: Result[McpCallToolResponse] | None = None + while True: + last = self.call_tool(key, server_id=server_id, name=name, arguments=arguments) + if not _is_mcp_not_synced(last, tool_name=name): + return unwrap(last) + if time.monotonic() >= deadline: + raise AssertionError( + f"tools/call for {name!r} on server {server_id} still missing on the " + f"data plane after {self.proxy.poll_timeout}s (multi-worker registry lag); " + f"last result: {last}" + ) + time.sleep(self.proxy.poll_interval) + + def await_call_tool_denied( + self, + key: str, + *, + server_id: str, + name: str, + arguments: McpToolArguments, + ) -> UnknownApiError: + """Poll tools/call until a cold-worker miss clears and the call is 403 access_denied.""" + deadline = time.monotonic() + self.proxy.poll_timeout + last: Result[McpCallToolResponse] | None = None + while True: + last = self.call_tool(key, server_id=server_id, name=name, arguments=arguments) + if isinstance(last, UnknownApiError) and last.status_code == 403: + return last + if not _is_mcp_not_synced(last, tool_name=name): + raise AssertionError( + f"ungranted key's tools/call was not 403 access_denied: {last}" + ) + if time.monotonic() >= deadline: + raise AssertionError( + f"ungranted key never got 403 for {name!r} within {self.proxy.poll_timeout}s; " + f"last result: {last}" + ) + time.sleep(self.proxy.poll_interval) + def register_mcp_content_filter(self, *, name: str, blocked_keyword: str) -> str: """Register a default-on content-filter guardrail that runs on the MCP tool-call hook (pre_mcp_call) and blocks a single keyword. The keyword is @@ -317,5 +372,39 @@ class McpClient: ) +def _is_mcp_not_synced( + result: Result[McpCallToolResponse], + *, + tool_name: str | None = None, +) -> bool: + """True only for gateway multi-worker registry misses, not upstream errors. + + Matches the proxy's own shapes: + - ValueError ``Tool not found`` wrapped as HTTP 500 (cold tool map / + unresolved server on this process) + - REST ``server_not_found`` when this worker has not loaded the MCP server row + + Does not treat arbitrary 500 bodies that merely mention "tool" and "not found" + (e.g. upstream MCP payload text) as lag, so await_call_tool does not retry + real failures or non-idempotent calls. + """ + if not isinstance(result, UnknownApiError) or result.status_code != 500: + return False + body = result.body + body_l = body.lower() + + if "server_not_found" in body_l: + return True + if re.search(r"mcp server ['\"][^'\"]+['\"] was not found", body_l): + return True + + # Gateway: "Tool search_datadog_logs not found" (optionally inside a longer message) + if tool_name is not None: + return ( + re.search(rf"\btool\s+{re.escape(tool_name)}\s+not found\b", body_l) is not None + ) + return re.search(r"\btool\s+\S+\s+not found\b", body_l) is not None + + def build_client(proxy: ProxyClient) -> McpClient: return McpClient(proxy=proxy) diff --git a/tests/e2e/mcp/test_mcp_access_group_e2e.py b/tests/e2e/mcp/test_mcp_access_group_e2e.py index 1b53d1ca0b4..f72b75fd43d 100644 --- a/tests/e2e/mcp/test_mcp_access_group_e2e.py +++ b/tests/e2e/mcp/test_mcp_access_group_e2e.py @@ -29,6 +29,7 @@ class TestMcpAccessGroupToolSelection: ) -> None: group = f"e2e-mcp-grp-{unique_marker()}" server_id = register_datadog_mcp(client, resources, mcp_access_groups=[group]) + client.await_registered(server_id) granted = client.generate_key( user_id=f"e2e-mcp-ag-granted-{unique_marker()}", diff --git a/tests/e2e/mcp/test_mcp_datadog_e2e.py b/tests/e2e/mcp/test_mcp_datadog_e2e.py index 8a539b86bff..d093e307f99 100644 --- a/tests/e2e/mcp/test_mcp_datadog_e2e.py +++ b/tests/e2e/mcp/test_mcp_datadog_e2e.py @@ -60,6 +60,7 @@ class TestDatadogMcpRoundTrip: _assert_datadog_logger_active(client.proxy) server_id = register_datadog_mcp(client, resources) + client.await_registered(server_id) marker = f"{MARKER_PREFIX}{unique_marker()}" key = client.generate_key( @@ -78,22 +79,19 @@ class TestDatadogMcpRoundTrip: ) tool_name = client.await_tool(key, server_id, SEARCH_LOGS_TOOL) - - call = unwrap( - client.call_tool( - key, - server_id=server_id, - name=tool_name, - arguments={ - "query": marker, - "from": DD_SEARCH_FROM, - "to": "now", - "max_tokens": 5000, - "telemetry": { - "intent": "e2e assert seeded litellm completion log is searchable via MCP" - }, + call = client.await_call_tool( + key, + server_id=server_id, + name=tool_name, + arguments={ + "query": marker, + "from": DD_SEARCH_FROM, + "to": "now", + "max_tokens": 5000, + "telemetry": { + "intent": "e2e assert seeded litellm completion log is searchable via MCP" }, - ) + }, ) assert call.is_error is not True, f"search_datadog_logs errored: {call}" body = call.all_text diff --git a/tests/e2e/mcp/test_mcp_key_access_e2e.py b/tests/e2e/mcp/test_mcp_key_access_e2e.py index 35c864c07d8..678424e36d1 100644 --- a/tests/e2e/mcp/test_mcp_key_access_e2e.py +++ b/tests/e2e/mcp/test_mcp_key_access_e2e.py @@ -16,7 +16,7 @@ import pytest from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp from e2e_config import DD_SEARCH_FROM, unique_marker -from e2e_http import UnknownApiError, unwrap +from e2e_http import unwrap from lifecycle import ResourceManager from mcp_client import McpClient @@ -72,13 +72,12 @@ class TestMcpKeyWithoutAccessIsDenied: "max_tokens": 1000, "telemetry": {"intent": "e2e control call proving granted key can invoke Datadog MCP"}, } - permitted_call = unwrap( - client.call_tool(permitted_key, server_id=server_id, name=tool_name, arguments=search_args) + permitted_call = client.await_call_tool( + permitted_key, server_id=server_id, name=tool_name, arguments=search_args ) assert permitted_call.is_error is not True, f"granted key's tool call errored: {permitted_call}" - match client.call_tool(denied_key, server_id=server_id, name=tool_name, arguments=search_args): - case UnknownApiError(status_code=403, body=body): - assert "access_denied" in body, f"403 was not an MCP access denial: {body}" - case other: - pytest.fail(f"ungranted key's tool call was not refused with 403 access_denied: {other}") + denied = client.await_call_tool_denied( + denied_key, server_id=server_id, name=tool_name, arguments=search_args + ) + assert "access_denied" in denied.body, f"403 was not an MCP access denial: {denied.body}" From 6cfcb6cd839c1d23c7a59f247b1900346b9b2cab Mon Sep 17 00:00:00 2001 From: milan Date: Wed, 29 Jul 2026 14:08:18 +0000 Subject: [PATCH 013/672] fix(vertex_ai): translate /v1/embeddings batch rows to Gemini embedding shape Vertex batch files sent every jsonl line through the generateContent transform, so embeddings rows went out as {"request": {"contents": [...]}} and Vertex rejected each one with "no such field: 'contents'"; the OpenAI "input" was dropped along the way too. Route lines by their own url: embeddings lines now emit the EmbedContentRequest shape (singular content, embed_content_config sibling, custom_id round-tripping through the top-level key), and matching output rows come back as OpenAI embeddings responses. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/vertex_ai/files/transformation.py | 253 +++++++++++++--- .../test_vertex_ai_files_transformation.py | 277 ++++++++++++++++++ 2 files changed, 491 insertions(+), 39 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index dd877b52eb8..516a3ca7184 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -5,6 +5,7 @@ import json import os import re import time +from collections.abc import Mapping from typing import ( Any, Callable, @@ -16,6 +17,7 @@ from typing import ( Tuple, Union, ) +from urllib.parse import unquote import httpx from httpx import Headers, Response @@ -51,6 +53,9 @@ from litellm.llms.vertex_ai.gemini.transformation import _transform_request_body from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) +from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( + transform_openai_input_gemini_embed_content, +) from litellm.types.llms.openai import ( AllMessageValues, CreateFileRequest, @@ -62,13 +67,26 @@ from litellm.types.llms.openai import ( ) from litellm.types.files import StreamingMediaUploadConfig from litellm.types.llms.vertex_ai import GcsBucketResponse -from litellm.types.utils import LlmProviders, ModelResponse +from litellm.types.utils import ( + Embedding, + EmbeddingResponse, + LlmProviders, + ModelResponse, + Usage, +) from ..common_utils import VertexAIError from ..vertex_llm_base import VertexBase _GCP_LABEL_VALUE_MAX_LEN = 63 _CUSTOM_ID_RAW_LABEL_PREFIX = "b32_" +_VERTEX_BATCH_KEY_FIELD = "key" +_MANAGED_GCS_MODEL_PATH_PATTERN = re.compile(r"publishers/[^/]+/models/([^/?]+)") +_EMBED_CONTENT_CONFIG_FIELD_BY_GEMINI_PARAM = { + "outputDimensionality": "output_dimensionality", + "taskType": "task_type", + "title": "title", +} def _sanitize_gcp_label_value(value: str) -> str: @@ -131,6 +149,21 @@ def _set_litellm_batch_custom_id_labels(labels: Dict[str, str], custom_id: Any) labels[f"litellm_custom_id_raw_{index}"] = raw_label_chunk +def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, Any]) -> str: + """ + Resolve the OpenAI `custom_id` for a Vertex batch output row. + + Embedding rows carry it in the top-level `key` field that Vertex echoes back; + `generateContent` rows have no such field, so it is smuggled through request + labels instead (see `_set_litellm_batch_custom_id_labels`). + """ + key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) + if key is not None: + return str(key) + request_data = vertex_output_row.get("request") or {} + return _get_litellm_batch_custom_id_from_labels(request_data.get("labels") or {}) + + def _get_litellm_batch_custom_id_from_labels(labels: Dict[str, Any]) -> str: """Prefer encoded custom_id when present (see _set_litellm_batch_custom_id_labels).""" raw = labels.get("litellm_custom_id_raw") @@ -149,10 +182,156 @@ def _get_litellm_batch_custom_id_from_labels(labels: Dict[str, Any]) -> str: return str(labels.get("litellm_custom_id", "unknown")) +def _is_vertex_embeddings_batch_output_row(vertex_output_row: Mapping[str, Any]) -> bool: + """ + Whether a Vertex batch output row came from an `EmbedContentRequest`. + + Successful rows hold the vector under `response.embedding.values`; failed rows only + carry `status`, so they are recognized from the singular `content` that the + embeddings request shape echoes back. + """ + if "request" not in vertex_output_row: + return False + response = vertex_output_row.get("response") + if isinstance(response, dict) and isinstance(response.get("embedding"), dict): + return True + request_data = vertex_output_row.get("request") + return bool(vertex_output_row.get("status")) and isinstance(request_data, dict) and "content" in request_data + + +def _openai_batch_output_row( + custom_id: str, + body: Mapping[str, Any] | None = None, + error: Mapping[str, str] | None = None, +) -> Mapping[str, Any]: + """ + One row of an OpenAI batch output file. Per the OpenAI Batch spec, failed rows set + `response` to null and populate `error` instead. + """ + return { + "id": f"batch_req_{uuid.uuid4()}", + "custom_id": custom_id, + "response": None + if body is None + else { + "status_code": 200, + "request_id": body.get("id", ""), + "body": body, + }, + "error": error, + } + + +def _transform_vertex_embeddings_batch_output_row_to_openai( + vertex_output_row: Mapping[str, Any], + model: str | None, +) -> Mapping[str, Any]: + """ + Transforms one Vertex Gemini Embedding batch output row into an OpenAI batch + output row holding an `/v1/embeddings` response body. + + Example Vertex jsonl + {"key": "id_1", "request": {...}, "response": {"tokenCount": "2", "embedding": {"values": [-0.015, 0.024]}}} + + `tokenCount` is serialized as a string by Vertex (int64 proto field), and the row + carries no `modelVersion`, so the model comes from the batch the row belongs to. + """ + custom_id = _get_litellm_batch_custom_id(vertex_output_row) + status = vertex_output_row.get("status", "") + if status: + return _openai_batch_output_row( + custom_id=custom_id, + error={"code": "vertex_ai_error", "message": status}, + ) + + vertex_response = vertex_output_row.get("response") or {} + token_count = int(vertex_response.get("tokenCount") or 0) + body = EmbeddingResponse( + model=model or "", + data=[ + Embedding( + embedding=vertex_response["embedding"]["values"], + index=0, + object="embedding", + ) + ], + usage=Usage(prompt_tokens=token_count, total_tokens=token_count), + ).model_dump() + return _openai_batch_output_row(custom_id=custom_id, body=body) + + +def _model_from_managed_gcs_url(url: str) -> str | None: + """ + Extracts the model from a LiteLLM-managed Vertex batch GCS url. + + Batch inputs and their sibling outputs are stored under + `.../publishers/google/models//...`, which is the only place the model of an + embeddings batch output row can be recovered from; unlike `generateContent` + responses, embedding rows carry no `modelVersion`. + """ + match = _MANAGED_GCS_MODEL_PATH_PATTERN.search(unquote(url)) + return match.group(1) if match else None + + +def _is_embeddings_batch_entry(openai_entry: Mapping[str, Any]) -> bool: + """ + Whether an OpenAI batch JSONL line targets the embeddings endpoint. + + OpenAI puts the target route on each line's `url` (e.g. `/v1/embeddings`); Vertex + has no equivalent per-line field, so the route decides which Vertex request shape + the line has to be translated into. + """ + url = openai_entry.get("url") + if not isinstance(url, str): + return False + path = url.split("?")[0].rstrip("/") + return path == "embeddings" or path.endswith("/embeddings") + + +def _openai_batch_jsonl_entry_to_vertex_embeddings_row( + openai_entry: Mapping[str, Any], +) -> Mapping[str, Any]: + """ + Transforms a single OpenAI `/v1/embeddings` batch entry into a Vertex Gemini + Embedding batch row. + + Example Vertex jsonl + {"key": "id_1", "request": {"content": {"parts": [{"text": "Hello World"}]}}, "embed_content_config": {"output_dimensionality": 768, "task_type": "RETRIEVAL_DOCUMENT"}} + + Note that `content` is singular (an `EmbedContentRequest`, not a + `GenerateContentRequest`), the per-row config is a sibling of `request` rather than + part of it, and the `custom_id` round-trips through the top-level `key`. + + API Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/batch-prediction-genai-embeddings + """ + openai_request_body = openai_entry.get("body") or {} + embedding_input = openai_request_body.get("input") + if embedding_input is None: + raise ValueError("`input` is required on /v1/embeddings batch requests, but was not provided") + + embed_content_request = transform_openai_input_gemini_embed_content( + input=embedding_input, + model=openai_request_body.get("model", ""), + optional_params=openai_request_body, + ) + embed_content_config = { + config_field: embed_content_request[gemini_param] + for gemini_param, config_field in _EMBED_CONTENT_CONFIG_FIELD_BY_GEMINI_PARAM.items() + if gemini_param in embed_content_request + } + + custom_id = openai_entry.get("custom_id") + return { + **({_VERTEX_BATCH_KEY_FIELD: str(custom_id)} if custom_id is not None else {}), + "request": {"content": embed_content_request["content"]}, + **({"embed_content_config": embed_content_config} if embed_content_config else {}), + } + + def _openai_batch_jsonl_entry_to_vertex_wrapped_request( openai_entry: Dict[str, Any], map_openai_to_vertex_params: Callable[[Dict[str, Any]], Dict[str, Any]], -) -> Dict[str, Any]: +) -> Mapping[str, Any]: """ Transforms a single OpenAI JSONL batch entry into its Vertex wrapped request. @@ -160,6 +339,9 @@ def _openai_batch_jsonl_entry_to_vertex_wrapped_request( Example Vertex jsonl {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}} """ + if _is_embeddings_batch_entry(openai_entry): + return _openai_batch_jsonl_entry_to_vertex_embeddings_row(openai_entry) + openai_request_body = openai_entry.get("body") or {} vertex_request_body = _transform_request_body( messages=openai_request_body.get("messages", []), @@ -629,6 +811,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): transformed_content = self._try_transform_vertex_batch_output_to_openai( content=content, logging_obj=logging_obj, + model=_model_from_managed_gcs_url(str(raw_response.request.url)), ) if transformed_content != content: # Create a new response with transformed content and updated Content-Length @@ -650,7 +833,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): return HttpxBinaryResponseContent(response=raw_response) def _try_transform_vertex_batch_output_to_openai( - self, content: bytes, logging_obj: Optional[LiteLLMLoggingObj] = None + self, + content: bytes, + logging_obj: LiteLLMLoggingObj | None = None, + model: str | None = None, ) -> bytes: """ Try to transform Vertex AI batch output to OpenAI format. @@ -692,7 +878,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): # first line is not valid UTF-8/JSON) raises and falls through to the # passthrough below, leaving the content untouched. first_row = json.loads(first_line) - is_vertex_batch_output = ( + is_vertex_batch_output = _is_vertex_embeddings_batch_output_row(first_row) or ( "request" in first_row and "response" in first_row and "processed_time" in first_row @@ -731,11 +917,19 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): output = bytearray() for line in itertools.chain([first_line], lines): try: - openai_output = self._transform_single_vertex_batch_output_to_openai( - vertex_output=json.loads(line), - vertex_gemini_config=vertex_gemini_config, - logging_obj=batch_transform_logging_obj, - mock_httpx_response=mock_httpx_response, + vertex_output_row = json.loads(line) + openai_output = ( + _transform_vertex_embeddings_batch_output_row_to_openai( + vertex_output_row=vertex_output_row, + model=model, + ) + if _is_vertex_embeddings_batch_output_row(vertex_output_row) + else self._transform_single_vertex_batch_output_to_openai( + vertex_output=vertex_output_row, + vertex_gemini_config=vertex_gemini_config, + logging_obj=batch_transform_logging_obj, + mock_httpx_response=mock_httpx_response, + ) ) except Exception: return content @@ -755,30 +949,22 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): vertex_gemini_config: VertexGeminiConfig, logging_obj: Logging, mock_httpx_response: httpx.Response, - ) -> Dict[str, Any]: + ) -> Mapping[str, Any]: """ Transform a single Vertex AI batch output line to OpenAI format. Uses the existing VertexGeminiConfig transformation for the response. """ - # Extract custom_id from request labels (prefer raw for OpenAI round-trip) - request_data = vertex_output.get("request", {}) - labels = request_data.get("labels", {}) or {} - custom_id = _get_litellm_batch_custom_id_from_labels(labels) + custom_id = _get_litellm_batch_custom_id(vertex_output) # Check if there's an error status = vertex_output.get("status", "") has_error = bool(status) if has_error: - return { - "id": f"batch_req_{uuid.uuid4()}", - "custom_id": custom_id, - "response": None, - "error": { - "code": "vertex_ai_error", - "message": status, - }, - } + return _openai_batch_output_row( + custom_id=custom_id, + error={"code": "vertex_ai_error", "message": status}, + ) # Transform successful response using existing transformation vertex_response = vertex_output.get("response", {}) @@ -804,24 +990,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): response_dict = transformed_response.model_dump() # Return in OpenAI batch format - return { - "id": f"batch_req_{uuid.uuid4()}", - "custom_id": custom_id, - "response": { - "status_code": 200, - "request_id": response_dict.get("id", ""), - "body": response_dict, - }, - "error": None, - } + return _openai_batch_output_row(custom_id=custom_id, body=response_dict) except Exception as e: - return { - "id": f"batch_req_{uuid.uuid4()}", - "custom_id": custom_id, - "response": None, - "error": { + return _openai_batch_output_row( + custom_id=custom_id, + error={ "code": "transformation_error", "message": f"Failed to transform response: {str(e)}", }, - } + ) diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 8c5305ee67b..636a3106617 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -1318,3 +1318,280 @@ class TestConfiguredBucketNameResolution: assert "bucket_name" in OPTIONAL_KWARGS_KEYS params = get_litellm_params(bucket_name="my-legacy-bucket") assert params.get("bucket_name") == "my-legacy-bucket" + + +def _embeddings_entry(**overrides): + entry = { + "custom_id": "request-1", + "method": "POST", + "url": "/v1/embeddings", + "body": {"model": "gemini-embedding-2", "input": "hello world"}, + } + entry.update(overrides) + return entry + + +class TestVertexEmbeddingsBatchInputTranslation: + """ + /v1/embeddings batch lines must be translated to Vertex's Gemini Embedding batch + shape, not the generateContent shape. + + Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/batch-prediction-genai-embeddings + """ + + def test_should_emit_embed_content_request_shape(self): + (row,) = _wrap_entries([_embeddings_entry()]) + + assert row["request"] == {"content": {"parts": [{"text": "hello world"}]}} + assert "contents" not in row["request"] + assert "labels" not in row["request"] + + def test_should_round_trip_custom_id_through_top_level_key(self): + (row,) = _wrap_entries([_embeddings_entry(custom_id="MyRequest-1")]) + + assert row["key"] == "MyRequest-1" + + def test_should_omit_key_when_no_custom_id(self): + entry = _embeddings_entry() + del entry["custom_id"] + + (row,) = _wrap_entries([entry]) + + assert "key" not in row + + def test_should_map_openai_params_to_embed_content_config_sibling(self): + (row,) = _wrap_entries( + [ + _embeddings_entry( + body={ + "model": "gemini-embedding-001", + "input": "hello world", + "dimensions": 768, + "task_type": "RETRIEVAL_DOCUMENT", + "title": "some_title", + } + ) + ] + ) + + assert row["embed_content_config"] == { + "output_dimensionality": 768, + "task_type": "RETRIEVAL_DOCUMENT", + "title": "some_title", + } + assert "output_dimensionality" not in row["request"] + + def test_should_omit_embed_content_config_when_no_params_given(self): + (row,) = _wrap_entries([_embeddings_entry()]) + + assert "embed_content_config" not in row + + def test_should_translate_multimodal_gcs_input(self): + (row,) = _wrap_entries( + [ + _embeddings_entry( + body={ + "model": "gemini-embedding-2", + "input": "gs://cloud-samples-data/generative-ai/image/benchmark.jpeg", + } + ) + ] + ) + + assert row["request"]["content"]["parts"] == [ + { + "file_data": { + "mime_type": "image/jpeg", + "file_uri": "gs://cloud-samples-data/generative-ai/image/benchmark.jpeg", + } + } + ] + + @pytest.mark.parametrize("url", ["/v1/embeddings", "v1/embeddings", "/v1/embeddings/"]) + def test_should_detect_embeddings_route_variants(self, url): + (row,) = _wrap_entries([_embeddings_entry(url=url)]) + + assert "content" in row["request"] + + def test_should_raise_when_input_missing(self): + with pytest.raises(ValueError, match="`input` is required"): + _wrap_entries([_embeddings_entry(body={"model": "gemini-embedding-2"})]) + + def test_should_keep_chat_completions_lines_on_generate_content_path(self): + (row,) = _wrap_entries( + [ + { + "custom_id": "request-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "gemini-2.0-flash-001", + "messages": [{"role": "user", "content": "Hello"}], + }, + } + ] + ) + + assert row["request"]["contents"] == [ + {"role": "user", "parts": [{"text": "Hello"}]} + ] + assert row["request"]["labels"]["litellm_custom_id"] == "request-1" + assert "key" not in row + + def test_should_translate_each_line_by_its_own_url(self): + chat_row, embeddings_row = _wrap_entries( + [ + { + "custom_id": "chat-1", + "url": "/v1/chat/completions", + "body": { + "model": "gemini-2.0-flash-001", + "messages": [{"role": "user", "content": "Hello"}], + }, + }, + _embeddings_entry(custom_id="embed-1"), + ] + ) + + assert "contents" in chat_row["request"] + assert "content" in embeddings_row["request"] + + +class TestVertexEmbeddingsBatchOutputTranslation: + """Vertex Gemini Embedding batch output rows must come back as OpenAI batch rows.""" + + def _vertex_embeddings_output_row(self, **overrides): + row = { + "key": "request-1", + "request": {"content": {"parts": [{"text": "hello world"}]}}, + "response": { + "tokenCount": "2", + "embedding": {"values": [-0.015, 0.024]}, + }, + } + row.update(overrides) + return row + + def _transform(self, config, rows, url="https://example.com"): + content = "\n".join(json.dumps(row) for row in rows).encode("utf-8") + result = config.transform_file_content_response( + raw_response=httpx.Response( + status_code=200, + content=content, + headers={"content-type": "application/octet-stream"}, + request=httpx.Request("GET", url), + ), + logging_obj=MagicMock(), + litellm_params={}, + ) + return [ + json.loads(line) + for line in result.response.content.decode("utf-8").split("\n") + ] + + def test_should_transform_embeddings_output_to_openai_batch_row(self, config): + (result,) = self._transform(config, [self._vertex_embeddings_output_row()]) + + assert result["custom_id"] == "request-1" + assert result["error"] is None + assert result["response"]["status_code"] == 200 + body = result["response"]["body"] + assert body["object"] == "list" + assert body["data"] == [ + {"embedding": [-0.015, 0.024], "index": 0, "object": "embedding"} + ] + assert body["usage"]["prompt_tokens"] == 2 + assert body["usage"]["total_tokens"] == 2 + + def test_should_resolve_model_from_managed_gcs_object_path(self, config): + object_path = urllib.parse.quote( + "litellm-vertex-files/publishers/google/models/gemini-embedding-2/" + "prediction-model-2026-07-29T05:55:52Z/predictions.jsonl", + safe="", + ) + url = f"https://storage.googleapis.com/storage/v1/b/my-bucket/o/{object_path}?alt=media" + + (result,) = self._transform( + config, [self._vertex_embeddings_output_row()], url=url + ) + + assert result["response"]["body"]["model"] == "gemini-embedding-2" + + def test_should_surface_failed_embeddings_row_as_error(self, config): + (result,) = self._transform( + config, + [ + self._vertex_embeddings_output_row( + status="Failed to parse JSON into proto", response={} + ) + ], + ) + + assert result["custom_id"] == "request-1" + assert result["response"] is None + assert result["error"]["code"] == "vertex_ai_error" + assert "Failed to parse JSON into proto" in result["error"]["message"] + + def test_should_transform_every_row_of_a_multi_row_file(self, config): + results = self._transform( + config, + [ + self._vertex_embeddings_output_row(key=f"request-{index}") + for index in range(3) + ], + ) + + assert [result["custom_id"] for result in results] == [ + "request-0", + "request-1", + "request-2", + ] + + def test_should_end_to_end_round_trip_openai_embeddings_batch(self, config): + (vertex_row,) = _wrap_entries( + [ + _embeddings_entry( + custom_id="MyRequest-1", + body={ + "model": "gemini-embedding-2", + "input": "hello world", + "dimensions": 2, + }, + ) + ] + ) + + (result,) = self._transform( + config, + [ + { + **vertex_row, + "status": "", + "processed_time": "2026-07-29T05:55:52.379528Z", + "response": { + "tokenCount": "2", + "embedding": {"values": [-0.015, 0.024]}, + }, + } + ], + ) + + assert result["custom_id"] == "MyRequest-1" + assert result["response"]["body"]["data"][0]["embedding"] == [-0.015, 0.024] + + def test_should_leave_legacy_predict_embeddings_output_untouched(self, config): + legacy_row = { + "instance": {"content": "hello world"}, + "predictions": [ + { + "embeddings": { + "statistics": {"token_count": 2, "truncated": False}, + "values": [0.2], + } + } + ], + "status": "", + } + content = json.dumps(legacy_row).encode("utf-8") + + assert config._try_transform_vertex_batch_output_to_openai(content) == content From e96614a39f45d8b6ffde5a3e2a05e63f6d73541d Mon Sep 17 00:00:00 2001 From: milan Date: Wed, 29 Jul 2026 14:44:25 +0000 Subject: [PATCH 014/672] fix(vertex_ai): put embed config inside the request and read live usage A live Vertex batch run showed the documented "embed_content_config" sibling of "request" is rejected by the API ("unsupported type"), failing the whole job rather than the row; the same fields inside the EmbedContentRequest succeed and honor output_dimensionality. Real output rows also report usage under response.usageMetadata.promptTokenCount, not the documented response.tokenCount, so every row came back with zero tokens. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/vertex_ai/files/transformation.py | 29 +++++++------ .../test_vertex_ai_files_transformation.py | 42 ++++++++++++++----- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 516a3ca7184..1a5d40dabb2 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -82,7 +82,7 @@ _GCP_LABEL_VALUE_MAX_LEN = 63 _CUSTOM_ID_RAW_LABEL_PREFIX = "b32_" _VERTEX_BATCH_KEY_FIELD = "key" _MANAGED_GCS_MODEL_PATH_PATTERN = re.compile(r"publishers/[^/]+/models/([^/?]+)") -_EMBED_CONTENT_CONFIG_FIELD_BY_GEMINI_PARAM = { +_EMBED_REQUEST_FIELD_BY_GEMINI_PARAM = { "outputDimensionality": "output_dimensionality", "taskType": "task_type", "title": "title", @@ -231,10 +231,11 @@ def _transform_vertex_embeddings_batch_output_row_to_openai( output row holding an `/v1/embeddings` response body. Example Vertex jsonl - {"key": "id_1", "request": {...}, "response": {"tokenCount": "2", "embedding": {"values": [-0.015, 0.024]}}} + {"key": "id_1", "request": {...}, "response": {"embedding": {"values": [-0.015, 0.024]}, "usageMetadata": {"promptTokenCount": 2}}} - `tokenCount` is serialized as a string by Vertex (int64 proto field), and the row - carries no `modelVersion`, so the model comes from the batch the row belongs to. + Live rows report usage under `usageMetadata`; the documented `tokenCount` is kept as + a fallback. The row carries no `modelVersion`, so the model comes from the batch it + belongs to. """ custom_id = _get_litellm_batch_custom_id(vertex_output_row) status = vertex_output_row.get("status", "") @@ -245,7 +246,8 @@ def _transform_vertex_embeddings_batch_output_row_to_openai( ) vertex_response = vertex_output_row.get("response") or {} - token_count = int(vertex_response.get("tokenCount") or 0) + usage_metadata = vertex_response.get("usageMetadata") or {} + token_count = int(usage_metadata.get("promptTokenCount") or vertex_response.get("tokenCount") or 0) body = EmbeddingResponse( model=model or "", data=[ @@ -296,11 +298,13 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_row( Embedding batch row. Example Vertex jsonl - {"key": "id_1", "request": {"content": {"parts": [{"text": "Hello World"}]}}, "embed_content_config": {"output_dimensionality": 768, "task_type": "RETRIEVAL_DOCUMENT"}} + {"key": "id_1", "request": {"content": {"parts": [{"text": "Hello World"}]}, "output_dimensionality": 768, "task_type": "RETRIEVAL_DOCUMENT"}} Note that `content` is singular (an `EmbedContentRequest`, not a - `GenerateContentRequest`), the per-row config is a sibling of `request` rather than - part of it, and the `custom_id` round-trips through the top-level `key`. + `GenerateContentRequest`) and that the `custom_id` round-trips through the top-level + `key`. The docs put the per-row config in an `embed_content_config` sibling of + `request`, but the API rejects that key outright and fails the whole batch job, so + the config fields go inside the `EmbedContentRequest` itself. API Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/batch-prediction-genai-embeddings """ @@ -314,17 +318,16 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_row( model=openai_request_body.get("model", ""), optional_params=openai_request_body, ) - embed_content_config = { - config_field: embed_content_request[gemini_param] - for gemini_param, config_field in _EMBED_CONTENT_CONFIG_FIELD_BY_GEMINI_PARAM.items() + embed_request_fields = { + request_field: embed_content_request[gemini_param] + for gemini_param, request_field in _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM.items() if gemini_param in embed_content_request } custom_id = openai_entry.get("custom_id") return { **({_VERTEX_BATCH_KEY_FIELD: str(custom_id)} if custom_id is not None else {}), - "request": {"content": embed_content_request["content"]}, - **({"embed_content_config": embed_content_config} if embed_content_config else {}), + "request": {"content": embed_content_request["content"], **embed_request_fields}, } diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 636a3106617..3eff3083220 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -1359,7 +1359,11 @@ class TestVertexEmbeddingsBatchInputTranslation: assert "key" not in row - def test_should_map_openai_params_to_embed_content_config_sibling(self): + def test_should_map_openai_params_into_the_embed_content_request(self): + """ + The docs put these in an `embed_content_config` sibling of `request`, but Vertex + rejects that key and fails the whole job, so they belong inside the request. + """ (row,) = _wrap_entries( [ _embeddings_entry( @@ -1374,17 +1378,20 @@ class TestVertexEmbeddingsBatchInputTranslation: ] ) - assert row["embed_content_config"] == { - "output_dimensionality": 768, - "task_type": "RETRIEVAL_DOCUMENT", - "title": "some_title", + assert row == { + "key": "request-1", + "request": { + "content": {"parts": [{"text": "hello world"}]}, + "output_dimensionality": 768, + "task_type": "RETRIEVAL_DOCUMENT", + "title": "some_title", + }, } - assert "output_dimensionality" not in row["request"] - def test_should_omit_embed_content_config_when_no_params_given(self): + def test_should_omit_config_fields_when_no_params_given(self): (row,) = _wrap_entries([_embeddings_entry()]) - assert "embed_content_config" not in row + assert set(row["request"]) == {"content"} def test_should_translate_multimodal_gcs_input(self): (row,) = _wrap_entries( @@ -1465,8 +1472,8 @@ class TestVertexEmbeddingsBatchOutputTranslation: "key": "request-1", "request": {"content": {"parts": [{"text": "hello world"}]}}, "response": { - "tokenCount": "2", "embedding": {"values": [-0.015, 0.024]}, + "usageMetadata": {"promptTokenCount": 2}, }, } row.update(overrides) @@ -1503,6 +1510,21 @@ class TestVertexEmbeddingsBatchOutputTranslation: assert body["usage"]["prompt_tokens"] == 2 assert body["usage"]["total_tokens"] == 2 + def test_should_fall_back_to_documented_token_count_field(self, config): + (result,) = self._transform( + config, + [ + self._vertex_embeddings_output_row( + response={ + "embedding": {"values": [-0.015, 0.024]}, + "tokenCount": "2", + } + ) + ], + ) + + assert result["response"]["body"]["usage"]["prompt_tokens"] == 2 + def test_should_resolve_model_from_managed_gcs_object_path(self, config): object_path = urllib.parse.quote( "litellm-vertex-files/publishers/google/models/gemini-embedding-2/" @@ -1569,8 +1591,8 @@ class TestVertexEmbeddingsBatchOutputTranslation: "status": "", "processed_time": "2026-07-29T05:55:52.379528Z", "response": { - "tokenCount": "2", "embedding": {"values": [-0.015, 0.024]}, + "usageMetadata": {"promptTokenCount": 2}, }, } ], From 627d2755da56d01a6f5838bec967005027b2f35d Mon Sep 17 00:00:00 2001 From: milan Date: Wed, 29 Jul 2026 15:06:58 +0000 Subject: [PATCH 015/672] fix(vertex_ai): fan array embeddings input out into one vertex row per element Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/vertex_ai/files/transformation.py | 221 +++++++++++++----- .../files/test_vertex_ai_files_streaming.py | 5 +- .../test_vertex_ai_files_transformation.py | 180 +++++++++++++- 3 files changed, 339 insertions(+), 67 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 1a5d40dabb2..78a16b002e7 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -66,7 +66,7 @@ from litellm.types.llms.openai import ( PathLike, ) from litellm.types.files import StreamingMediaUploadConfig -from litellm.types.llms.vertex_ai import GcsBucketResponse +from litellm.types.llms.vertex_ai import GcsBucketResponse, GeminiEmbeddingInput from litellm.types.utils import ( Embedding, EmbeddingResponse, @@ -87,6 +87,7 @@ _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM = { "taskType": "task_type", "title": "title", } +_VERTEX_BATCH_FANNED_OUT_KEY_PATTERN = re.compile(r"(?P.*)#(?P\d+)/(?P\d+)") def _sanitize_gcp_label_value(value: str) -> str: @@ -222,46 +223,93 @@ def _openai_batch_output_row( } -def _transform_vertex_embeddings_batch_output_row_to_openai( - vertex_output_row: Mapping[str, Any], +def _split_vertex_batch_key(vertex_output_row: Mapping[str, Any]) -> tuple[str, int]: + """ + Resolve `(custom_id, index within that custom_id)` for a Vertex batch output row. + + A `/v1/embeddings` entry whose `input` is an array fans out into one Vertex row per + element, tagged `#/` (see `_vertex_batch_embeddings_key`), + so the rows can be reassembled into a single OpenAI response. + """ + key = _get_litellm_batch_custom_id(vertex_output_row) + match = _VERTEX_BATCH_FANNED_OUT_KEY_PATTERN.fullmatch(key) + if match is None or int(match["total"]) < 2: + return key, 0 + return match["custom_id"], int(match["index"]) + + +def _vertex_embeddings_rows_to_openai_batch_output_row( + custom_id: str, + vertex_output_rows: tuple[Mapping[str, Any], ...], model: str | None, ) -> Mapping[str, Any]: """ - Transforms one Vertex Gemini Embedding batch output row into an OpenAI batch - output row holding an `/v1/embeddings` response body. + Transforms the Vertex Gemini Embedding batch output rows belonging to one OpenAI + batch entry into an OpenAI batch output row holding an `/v1/embeddings` response. Example Vertex jsonl {"key": "id_1", "request": {...}, "response": {"embedding": {"values": [-0.015, 0.024]}, "usageMetadata": {"promptTokenCount": 2}}} - Live rows report usage under `usageMetadata`; the documented `tokenCount` is kept as - a fallback. The row carries no `modelVersion`, so the model comes from the batch it - belongs to. + An entry that asked for several embeddings at once maps to several rows here, which + become the indexed elements of a single `data` array. One failed element fails the + whole entry, since an OpenAI batch row is either a response or an error. Live rows + report usage under `usageMetadata`; the documented `tokenCount` is kept as a + fallback. Rows carry no `modelVersion`, so the model comes from the batch they + belong to. """ - custom_id = _get_litellm_batch_custom_id(vertex_output_row) - status = vertex_output_row.get("status", "") + status = next((row["status"] for row in vertex_output_rows if row.get("status")), "") if status: return _openai_batch_output_row( custom_id=custom_id, error={"code": "vertex_ai_error", "message": status}, ) - vertex_response = vertex_output_row.get("response") or {} - usage_metadata = vertex_response.get("usageMetadata") or {} - token_count = int(usage_metadata.get("promptTokenCount") or vertex_response.get("tokenCount") or 0) + responses = tuple(row.get("response") or {} for row in vertex_output_rows) + token_count = sum( + int((response.get("usageMetadata") or {}).get("promptTokenCount") or response.get("tokenCount") or 0) + for response in responses + ) body = EmbeddingResponse( model=model or "", data=[ Embedding( - embedding=vertex_response["embedding"]["values"], - index=0, + embedding=response["embedding"]["values"], + index=index, object="embedding", ) + for index, response in enumerate(responses) ], usage=Usage(prompt_tokens=token_count, total_tokens=token_count), ).model_dump() return _openai_batch_output_row(custom_id=custom_id, body=body) +def _transform_vertex_embeddings_batch_output_to_openai( + vertex_output_rows: Iterable[Mapping[str, Any]], + model: str | None, +) -> tuple[Mapping[str, Any], ...]: + """ + Transforms a whole Vertex Gemini Embedding batch output into OpenAI batch output + rows, one per OpenAI batch entry, in the order the entries first appear. + + Rows are grouped rather than mapped one to one because a single entry can fan out + into several Vertex rows, and Vertex returns them in arbitrary order. + """ + keyed_rows = tuple((_split_vertex_batch_key(row), row) for row in vertex_output_rows) + grouped_rows = { + custom_id: tuple(row for _, row in group) + for custom_id, group in itertools.groupby(sorted(keyed_rows, key=lambda kr: kr[0]), key=lambda kr: kr[0][0]) + } + return tuple( + _vertex_embeddings_rows_to_openai_batch_output_row( + custom_id=custom_id, + vertex_output_rows=grouped_rows[custom_id], + model=model, + ) + for custom_id in dict.fromkeys(custom_id for (custom_id, _), _ in keyed_rows) + ) + + def _model_from_managed_gcs_url(url: str) -> str | None: """ Extracts the model from a LiteLLM-managed Vertex batch GCS url. @@ -290,21 +338,49 @@ def _is_embeddings_batch_entry(openai_entry: Mapping[str, Any]) -> bool: return path == "embeddings" or path.endswith("/embeddings") -def _openai_batch_jsonl_entry_to_vertex_embeddings_row( - openai_entry: Mapping[str, Any], -) -> Mapping[str, Any]: +def _openai_embedding_input_elements( + embedding_input: GeminiEmbeddingInput, +) -> tuple[Union[str, List[str]], ...]: """ - Transforms a single OpenAI `/v1/embeddings` batch entry into a Vertex Gemini - Embedding batch row. + Split an OpenAI `input` into the elements that each get their own embedding. + + A string is one embedding, a flat array is one embedding per element, and a nested + array is one combined embedding per inner array, matching the online + `batchEmbedContents` path. + """ + if isinstance(embedding_input, list): + return tuple(embedding_input) + return (embedding_input,) + + +def _vertex_batch_embeddings_key(custom_id: str, index: int, total: int) -> str: + """ + The top-level `key` Vertex echoes back on an embeddings row. + + An entry asking for several embeddings needs several Vertex rows, so its key also + carries the element index and the group size; `_split_vertex_batch_key` reads them + back out. Entries asking for a single embedding keep their bare `custom_id`. + """ + return custom_id if total < 2 else f"{custom_id}#{index}/{total}" + + +def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( + openai_entry: Mapping[str, Any], +) -> tuple[Mapping[str, Any], ...]: + """ + Transforms a single OpenAI `/v1/embeddings` batch entry into Vertex Gemini Embedding + batch rows, one per requested embedding. Example Vertex jsonl {"key": "id_1", "request": {"content": {"parts": [{"text": "Hello World"}]}, "output_dimensionality": 768, "task_type": "RETRIEVAL_DOCUMENT"}} Note that `content` is singular (an `EmbedContentRequest`, not a `GenerateContentRequest`) and that the `custom_id` round-trips through the top-level - `key`. The docs put the per-row config in an `embed_content_config` sibling of - `request`, but the API rejects that key outright and fails the whole batch job, so - the config fields go inside the `EmbedContentRequest` itself. + `key`. An `EmbedContentRequest` returns exactly one vector, so an entry whose `input` + is an array fans out into one row per element and is reassembled on the way back. + The docs put the per-row config in an `embed_content_config` sibling of `request`, + but the API rejects that key outright and fails the whole batch job, so the config + fields go inside the `EmbedContentRequest` itself. API Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/batch-prediction-genai-embeddings """ @@ -313,37 +389,58 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_row( if embedding_input is None: raise ValueError("`input` is required on /v1/embeddings batch requests, but was not provided") - embed_content_request = transform_openai_input_gemini_embed_content( - input=embedding_input, - model=openai_request_body.get("model", ""), - optional_params=openai_request_body, + elements = _openai_embedding_input_elements(embedding_input) + if not elements: + raise ValueError("`input` on /v1/embeddings batch requests must not be empty") + + embed_content_requests = tuple( + transform_openai_input_gemini_embed_content( + input=element, + model=openai_request_body.get("model", ""), + optional_params=openai_request_body, + ) + for element in elements ) - embed_request_fields = { - request_field: embed_content_request[gemini_param] - for gemini_param, request_field in _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM.items() - if gemini_param in embed_content_request - } - custom_id = openai_entry.get("custom_id") - return { - **({_VERTEX_BATCH_KEY_FIELD: str(custom_id)} if custom_id is not None else {}), - "request": {"content": embed_content_request["content"], **embed_request_fields}, - } + return tuple( + { + **( + {} + if custom_id is None + else { + _VERTEX_BATCH_KEY_FIELD: _vertex_batch_embeddings_key( + custom_id=str(custom_id), + index=index, + total=len(embed_content_requests), + ) + } + ), + "request": { + "content": embed_content_request["content"], + **{ + request_field: embed_content_request[gemini_param] + for gemini_param, request_field in _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM.items() + if gemini_param in embed_content_request + }, + }, + } + for index, embed_content_request in enumerate(embed_content_requests) + ) -def _openai_batch_jsonl_entry_to_vertex_wrapped_request( +def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: Dict[str, Any], map_openai_to_vertex_params: Callable[[Dict[str, Any]], Dict[str, Any]], -) -> Mapping[str, Any]: +) -> tuple[Mapping[str, Any], ...]: """ - Transforms a single OpenAI JSONL batch entry into its Vertex wrapped request. + Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. jsonl body for vertex is {"request": } Example Vertex jsonl {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}} """ if _is_embeddings_batch_entry(openai_entry): - return _openai_batch_jsonl_entry_to_vertex_embeddings_row(openai_entry) + return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry) openai_request_body = openai_entry.get("body") or {} vertex_request_body = _transform_request_body( @@ -361,7 +458,7 @@ def _openai_batch_jsonl_entry_to_vertex_wrapped_request( vertex_request_body["labels"] = {} _set_litellm_batch_custom_id_labels(vertex_request_body["labels"], custom_id) - return {"request": vertex_request_body} + return ({"request": vertex_request_body},) def _iter_stripped_lines(raw_lines: Iterable[Union[str, bytes]]) -> Iterator[str]: @@ -459,10 +556,10 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for entry in _iter_openai_jsonl_entries(self._openai_file_content): - wrapped = _openai_batch_jsonl_entry_to_vertex_wrapped_request(entry, self._map_openai_to_vertex_params) - prefix = b"" if first else b"\n" - first = False - yield prefix + json.dumps(wrapped).encode("utf-8") + for wrapped in _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params): + prefix = b"" if first else b"\n" + first = False + yield prefix + json.dumps(wrapped).encode("utf-8") def iter_bytes(self) -> Iterator[bytes]: return self._iter_vertex_jsonl_chunks() @@ -914,25 +1011,29 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): request=httpx.Request(method="POST", url="https://example.com"), ) + all_lines = itertools.chain([first_line], lines) + + # Embedding rows are grouped by `custom_id` rather than transformed one at a + # time, since an entry that asked for several embeddings comes back as + # several rows, in arbitrary order. + if _is_vertex_embeddings_batch_output_row(first_row): + openai_outputs = _transform_vertex_embeddings_batch_output_to_openai( + vertex_output_rows=(json.loads(line) for line in all_lines), + model=model, + ) + return b"\n".join(json.dumps(openai_output).encode("utf-8") for openai_output in openai_outputs) + # Transform each row straight into the output buffer, so peak memory # stays at ~one row plus the output. If any row fails, return the # original content unchanged. output = bytearray() - for line in itertools.chain([first_line], lines): + for line in all_lines: try: - vertex_output_row = json.loads(line) - openai_output = ( - _transform_vertex_embeddings_batch_output_row_to_openai( - vertex_output_row=vertex_output_row, - model=model, - ) - if _is_vertex_embeddings_batch_output_row(vertex_output_row) - else self._transform_single_vertex_batch_output_to_openai( - vertex_output=vertex_output_row, - vertex_gemini_config=vertex_gemini_config, - logging_obj=batch_transform_logging_obj, - mock_httpx_response=mock_httpx_response, - ) + openai_output = self._transform_single_vertex_batch_output_to_openai( + vertex_output=json.loads(line), + vertex_gemini_config=vertex_gemini_config, + logging_obj=batch_transform_logging_obj, + mock_httpx_response=mock_httpx_response, ) except Exception: return content diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_streaming.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_streaming.py index 2e3280c0ed1..957fc7dbcf4 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_streaming.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_streaming.py @@ -37,7 +37,7 @@ from litellm.llms.vertex_ai.files.transformation import ( _get_litellm_batch_custom_id_from_labels, _iter_openai_jsonl_entries, _iter_openai_jsonl_lines, - _openai_batch_jsonl_entry_to_vertex_wrapped_request, + _openai_batch_jsonl_entry_to_vertex_rows, ) from litellm.types.llms.openai import CreateFileRequest @@ -84,8 +84,9 @@ def _reference_vertex_jsonl_string(cfg: VertexAIFilesConfig, content: str) -> st transform, so the streaming path can be checked against it for parity.""" entries = [json.loads(line) for line in content.splitlines() if line.strip()] return "\n".join( - json.dumps(_openai_batch_jsonl_entry_to_vertex_wrapped_request(entry, cfg._map_openai_to_vertex_params)) + json.dumps(row) for entry in entries + for row in _openai_batch_jsonl_entry_to_vertex_rows(entry, cfg._map_openai_to_vertex_params) ) diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 3eff3083220..73d1d6eeb5a 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -15,7 +15,7 @@ from unittest.mock import MagicMock from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, - _openai_batch_jsonl_entry_to_vertex_wrapped_request, + _openai_batch_jsonl_entry_to_vertex_rows, _sanitize_gcp_label_value, ) from litellm.types.llms.openai import OpenAIFileObject, HttpxBinaryResponseContent @@ -1054,14 +1054,15 @@ class TestTryTransformDoesNotMutateCallerLoggingObj: def _wrap_entries(openai_jsonl_content): - """Vertex-wrapped requests for a list of OpenAI batch entries, built via the - live single-entry transform that the streaming upload path uses.""" + """Vertex rows for a list of OpenAI batch entries, built via the live + single-entry transform that the streaming upload path uses.""" cfg = VertexAIFilesConfig() return [ - _openai_batch_jsonl_entry_to_vertex_wrapped_request( + row + for entry in openai_jsonl_content + for row in _openai_batch_jsonl_entry_to_vertex_rows( entry, cfg._map_openai_to_vertex_params ) - for entry in openai_jsonl_content ] @@ -1424,6 +1425,86 @@ class TestVertexEmbeddingsBatchInputTranslation: with pytest.raises(ValueError, match="`input` is required"): _wrap_entries([_embeddings_entry(body={"model": "gemini-embedding-2"})]) + def test_should_raise_when_input_empty(self): + with pytest.raises(ValueError, match="must not be empty"): + _wrap_entries( + [_embeddings_entry(body={"model": "gemini-embedding-2", "input": []})] + ) + + def test_should_fan_an_input_array_out_into_one_row_per_element(self): + """ + An `EmbedContentRequest` returns exactly one vector, so an OpenAI entry asking + for several embeddings needs several Vertex rows. + """ + rows = _wrap_entries( + [ + _embeddings_entry( + body={ + "model": "gemini-embedding-001", + "input": ["first", "second"], + "dimensions": 768, + } + ) + ] + ) + + assert rows == [ + { + "key": "request-1#0/2", + "request": { + "content": {"parts": [{"text": "first"}]}, + "output_dimensionality": 768, + }, + }, + { + "key": "request-1#1/2", + "request": { + "content": {"parts": [{"text": "second"}]}, + "output_dimensionality": 768, + }, + }, + ] + + def test_should_keep_the_bare_custom_id_for_single_element_arrays(self): + (row,) = _wrap_entries( + [ + _embeddings_entry( + body={"model": "gemini-embedding-2", "input": ["only one"]} + ) + ] + ) + + assert row["key"] == "request-1" + + def test_should_combine_a_nested_input_into_one_multipart_row(self): + """Nested arrays are the combined-embedding shape, as on the online path.""" + (row,) = _wrap_entries( + [ + _embeddings_entry( + body={ + "model": "gemini-embedding-2", + "input": [ + [ + "a caption", + "gs://cloud-samples-data/generative-ai/image/benchmark.jpeg", + ] + ], + } + ) + ] + ) + + assert row["key"] == "request-1" + assert row["request"]["content"]["parts"] == [ + {"text": "a caption"}, + { + "file_data": { + "mime_type": "image/jpeg", + "file_uri": "gs://cloud-samples-data/generative-ai/image/benchmark.jpeg", + } + }, + ] + def test_should_keep_chat_completions_lines_on_generate_content_path(self): (row,) = _wrap_entries( [ @@ -1569,6 +1650,95 @@ class TestVertexEmbeddingsBatchOutputTranslation: "request-2", ] + def test_should_reassemble_a_fanned_out_input_array_into_one_row(self, config): + """Vertex returns the rows of one entry in arbitrary order.""" + (result,) = self._transform( + config, + [ + self._vertex_embeddings_output_row( + key="request-1#1/2", + response={ + "embedding": {"values": [0.3, 0.4]}, + "usageMetadata": {"promptTokenCount": 5}, + }, + ), + self._vertex_embeddings_output_row( + key="request-1#0/2", + response={ + "embedding": {"values": [0.1, 0.2]}, + "usageMetadata": {"promptTokenCount": 3}, + }, + ), + ], + ) + + assert result["custom_id"] == "request-1" + assert result["response"]["body"]["data"] == [ + {"embedding": [0.1, 0.2], "index": 0, "object": "embedding"}, + {"embedding": [0.3, 0.4], "index": 1, "object": "embedding"}, + ] + assert result["response"]["body"]["usage"]["prompt_tokens"] == 8 + + def test_should_keep_fanned_out_entries_apart_and_in_file_order(self, config): + results = self._transform( + config, + [ + self._vertex_embeddings_output_row(key="request-2#0/2"), + self._vertex_embeddings_output_row(key="request-1"), + self._vertex_embeddings_output_row(key="request-2#1/2"), + ], + ) + + assert [result["custom_id"] for result in results] == ["request-2", "request-1"] + assert len(results[0]["response"]["body"]["data"]) == 2 + assert len(results[1]["response"]["body"]["data"]) == 1 + + def test_should_fail_the_whole_entry_when_one_of_its_rows_failed(self, config): + """An OpenAI batch row is either a response or an error, never both.""" + (result,) = self._transform( + config, + [ + self._vertex_embeddings_output_row(key="request-1#0/2"), + self._vertex_embeddings_output_row( + key="request-1#1/2", status="Quota exceeded", response={} + ), + ], + ) + + assert result["custom_id"] == "request-1" + assert result["response"] is None + assert result["error"]["message"] == "Quota exceeded" + + def test_should_end_to_end_round_trip_a_fanned_out_embeddings_batch(self, config): + first_row, second_row = _wrap_entries( + [ + _embeddings_entry( + custom_id="MyRequest-1", + body={ + "model": "gemini-embedding-2", + "input": ["hello world", "goodbye world"], + }, + ) + ] + ) + + (result,) = self._transform( + config, + [ + { + **row, + "status": "", + "response": {"embedding": {"values": values}}, + } + for row, values in ((second_row, [0.3]), (first_row, [0.1])) + ], + ) + + assert result["custom_id"] == "MyRequest-1" + assert [ + embedding["embedding"] for embedding in result["response"]["body"]["data"] + ] == [[0.1], [0.3]] + def test_should_end_to_end_round_trip_openai_embeddings_batch(self, config): (vertex_row,) = _wrap_entries( [ From 3c979f0b471214929b38de0fe61573fc864b7906 Mon Sep 17 00:00:00 2001 From: milan Date: Wed, 29 Jul 2026 15:23:14 +0000 Subject: [PATCH 016/672] test(vertex_ai): cover batch lines without a url staying on the chat path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_vertex_ai_files_transformation.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 73d1d6eeb5a..aef7f8b4684 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -1526,6 +1526,24 @@ class TestVertexEmbeddingsBatchInputTranslation: assert row["request"]["labels"]["litellm_custom_id"] == "request-1" assert "key" not in row + def test_should_keep_lines_without_a_url_on_generate_content_path(self): + """`url` is optional on a batch line, and chat is the shape LiteLLM has always assumed.""" + (row,) = _wrap_entries( + [ + { + "custom_id": "request-1", + "body": { + "model": "gemini-2.0-flash-001", + "messages": [{"role": "user", "content": "Hello"}], + }, + } + ] + ) + + assert row["request"]["contents"] == [ + {"role": "user", "parts": [{"text": "Hello"}]} + ] + def test_should_translate_each_line_by_its_own_url(self): chat_row, embeddings_row = _wrap_entries( [ From bf723fa9c167f48731f68ebe6b3bcab7351f5a83 Mon Sep 17 00:00:00 2001 From: milan Date: Wed, 29 Jul 2026 20:50:27 +0000 Subject: [PATCH 017/672] fix(vertex_ai): percent-encode the custom_id in fanned-out vertex batch keys Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/vertex_ai/files/transformation.py | 29 ++++--- .../test_vertex_ai_files_transformation.py | 75 +++++++++++++++++++ 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 78a16b002e7..90fc5fae082 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -17,7 +17,7 @@ from typing import ( Tuple, Union, ) -from urllib.parse import unquote +from urllib.parse import quote, unquote import httpx from httpx import Headers, Response @@ -87,7 +87,7 @@ _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM = { "taskType": "task_type", "title": "title", } -_VERTEX_BATCH_FANNED_OUT_KEY_PATTERN = re.compile(r"(?P.*)#(?P\d+)/(?P\d+)") +_VERTEX_BATCH_FANNED_OUT_KEY_PATTERN = re.compile(r"(?P[^#]*)#(?P\d+)/(?P\d+)") def _sanitize_gcp_label_value(value: str) -> str: @@ -160,7 +160,7 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, Any]) -> str: """ key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) if key is not None: - return str(key) + return unquote(str(key)) request_data = vertex_output_row.get("request") or {} return _get_litellm_batch_custom_id_from_labels(request_data.get("labels") or {}) @@ -228,14 +228,17 @@ def _split_vertex_batch_key(vertex_output_row: Mapping[str, Any]) -> tuple[str, Resolve `(custom_id, index within that custom_id)` for a Vertex batch output row. A `/v1/embeddings` entry whose `input` is an array fans out into one Vertex row per - element, tagged `#/` (see `_vertex_batch_embeddings_key`), - so the rows can be reassembled into a single OpenAI response. + element, tagged `#/` (see + `_vertex_batch_embeddings_key`), so the rows can be reassembled into a single OpenAI + response. """ - key = _get_litellm_batch_custom_id(vertex_output_row) - match = _VERTEX_BATCH_FANNED_OUT_KEY_PATTERN.fullmatch(key) - if match is None or int(match["total"]) < 2: - return key, 0 - return match["custom_id"], int(match["index"]) + key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) + if key is None: + return _get_litellm_batch_custom_id(vertex_output_row), 0 + match = _VERTEX_BATCH_FANNED_OUT_KEY_PATTERN.fullmatch(str(key)) + if match is None: + return unquote(str(key)), 0 + return unquote(match["custom_id"]), int(match["index"]) def _vertex_embeddings_rows_to_openai_batch_output_row( @@ -359,9 +362,11 @@ def _vertex_batch_embeddings_key(custom_id: str, index: int, total: int) -> str: An entry asking for several embeddings needs several Vertex rows, so its key also carries the element index and the group size; `_split_vertex_batch_key` reads them - back out. Entries asking for a single embedding keep their bare `custom_id`. + back out. The `custom_id` is percent-encoded so that a customer one ending in + `#/` cannot be mistaken for that tag, which would merge two entries. """ - return custom_id if total < 2 else f"{custom_id}#{index}/{total}" + encoded_custom_id = quote(custom_id, safe="") + return encoded_custom_id if total < 2 else f"{encoded_custom_id}#{index}/{total}" def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index aef7f8b4684..f95a63e4421 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -1476,6 +1476,19 @@ class TestVertexEmbeddingsBatchInputTranslation: assert row["key"] == "request-1" + def test_should_encode_a_custom_id_that_looks_like_a_fan_out_tag(self): + """A customer custom_id ending in `#/` must not read back as fan-out metadata.""" + (row,) = _wrap_entries( + [ + _embeddings_entry( + custom_id="request-1#0/2", + body={"model": "gemini-embedding-2", "input": "hello world"}, + ) + ] + ) + + assert row["key"] == "request-1%230%2F2" + def test_should_combine_a_nested_input_into_one_multipart_row(self): """Nested arrays are the combined-embedding shape, as on the online path.""" (row,) = _wrap_entries( @@ -1711,6 +1724,68 @@ class TestVertexEmbeddingsBatchOutputTranslation: assert len(results[0]["response"]["body"]["data"]) == 2 assert len(results[1]["response"]["body"]["data"]) == 1 + def test_should_not_merge_an_entry_whose_custom_id_looks_like_a_fan_out_tag(self, config): + """`request-1#0/2` is a legal custom_id, and a distinct entry from `request-1`.""" + lookalike_row, plain_row = _wrap_entries( + [ + _embeddings_entry( + custom_id="request-1#0/2", + body={"model": "gemini-embedding-2", "input": "lookalike"}, + ), + _embeddings_entry( + custom_id="request-1", + body={"model": "gemini-embedding-2", "input": "plain"}, + ), + ] + ) + + results = self._transform( + config, + [ + {**row, "status": "", "response": {"embedding": {"values": values}}} + for row, values in ((lookalike_row, [0.1]), (plain_row, [0.2])) + ], + ) + + assert [result["custom_id"] for result in results] == [ + "request-1#0/2", + "request-1", + ] + assert [ + result["response"]["body"]["data"][0]["embedding"] for result in results + ] == [[0.1], [0.2]] + + def test_should_round_trip_a_fan_out_of_a_custom_id_holding_the_separator(self, config): + rows = _wrap_entries( + [ + _embeddings_entry( + custom_id="request#1/1", + body={ + "model": "gemini-embedding-2", + "input": ["first", "second"], + }, + ) + ] + ) + + assert [row["key"] for row in rows] == [ + "request%231%2F1#0/2", + "request%231%2F1#1/2", + ] + + (result,) = self._transform( + config, + [ + {**row, "status": "", "response": {"embedding": {"values": values}}} + for row, values in zip(reversed(rows), ([0.3], [0.1])) + ], + ) + + assert result["custom_id"] == "request#1/1" + assert [ + embedding["embedding"] for embedding in result["response"]["body"]["data"] + ] == [[0.1], [0.3]] + def test_should_fail_the_whole_entry_when_one_of_its_rows_failed(self, config): """An OpenAI batch row is either a response or an error, never both.""" (result,) = self._transform( From 4429742e834377b666cc255ea0bceca22f2dc7b0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:11:44 +0000 Subject: [PATCH 018/672] fix(proxy): fetch background responses through the router in CheckResponsesCost Closes #35131 --- .../common_utils/check_responses_cost.py | 49 ++-- .../test_check_responses_cost.py | 216 ++++++++++++++++++ 2 files changed, 250 insertions(+), 15 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index dc0168683c8..5a587de12e9 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -1,10 +1,10 @@ """ Polls LiteLLM_ManagedObjectTable to check if the response is complete. -Cost tracking is handled automatically by litellm.aget_responses(). +Cost tracking is handled automatically by the get-responses call. """ from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional, cast import litellm from litellm._logging import verbose_proxy_logger @@ -13,11 +13,15 @@ from litellm.constants import ( MAX_OBJECTS_PER_POLL_CYCLE, STALE_OBJECT_CLEANUP_BATCH_SIZE, ) +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router +TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"}) + class CheckResponsesCost: def __init__( @@ -33,6 +37,28 @@ class CheckResponsesCost: self.prisma_client: PrismaClient = prisma_client self.llm_router: Router = llm_router + async def _get_response( + self, + response_id: str, + litellm_metadata: Dict[str, str], + ) -> ResponsesAPIResponse: + """Fetch the upstream response, using deployment credentials when available. + + LiteLLM-encoded response IDs carry the ``model_id`` of the deployment that + served the original request, so routing through ``llm_router`` applies that + deployment's ``api_base`` / ``api_key`` / ``api_version``, exactly like + ``GET /v1/responses/{id}`` does. ``litellm.aget_responses`` on its own only + sees provider env vars, so it fails for every deployment whose credentials + live in the config; the row then never leaves ``queued``. + """ + model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) + if model_id is None: + return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) + router_response = await self.llm_router.aget_responses( + response_id=response_id, litellm_metadata=litellm_metadata + ) + return cast(ResponsesAPIResponse, router_response) + async def _expire_stale_rows( self, cutoff: datetime, batch_size: int ) -> int: @@ -87,8 +113,8 @@ class CheckResponsesCost: Check if background responses are complete and track their cost. - Get all status="queued" or "in_progress" and file_purpose="response" jobs - Query the provider to check if response is complete - - Cost is automatically tracked by litellm.aget_responses() - - Mark completed/failed/cancelled responses as complete in the database + - Cost is automatically tracked by the get-responses call + - Mark responses in a terminal state as complete in the database """ try: await self._cleanup_stale_managed_objects() @@ -134,7 +160,7 @@ class CheckResponsesCost: litellm_metadata["model"] = model_name litellm_metadata["model_group"] = model_name # Use same value for model_group - response = await litellm.aget_responses( + response = await self._get_response( response_id=responses_id_security, litellm_metadata=litellm_metadata, ) @@ -144,21 +170,14 @@ class CheckResponsesCost: ) except Exception as e: - verbose_proxy_logger.info( + verbose_proxy_logger.warning( f"Skipping job {unified_object_id} due to error: {e}" ) continue - # Check if response is in a terminal state - if response.status == "completed": + if response.status in TERMINAL_RESPONSE_STATUSES: verbose_proxy_logger.info( - f"Response {unified_object_id} is complete. Cost automatically tracked by aget_responses." - ) - completed_jobs.append(job) - - elif response.status in ["failed", "cancelled"]: - verbose_proxy_logger.info( - f"Response {unified_object_id} has status {response.status}, marking as complete" + f"Response {unified_object_id} has terminal status {response.status}, marking as complete" ) completed_jobs.append(job) diff --git a/tests/proxy_unit_tests/test_check_responses_cost.py b/tests/proxy_unit_tests/test_check_responses_cost.py index 4c0ca94df48..16ad5c07919 100644 --- a/tests/proxy_unit_tests/test_check_responses_cost.py +++ b/tests/proxy_unit_tests/test_check_responses_cost.py @@ -449,6 +449,222 @@ class TestCheckResponsesCost: assert "job-3" in completion_call[1]["where"]["id"]["in"] assert "job-2" not in completion_call[1]["where"]["id"]["in"] + @pytest.mark.asyncio + async def test_encoded_response_id_is_fetched_through_router( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """ + Regression test for https://github.com/BerriAI/litellm/issues/35131 + + A background response created against a deployment whose credentials only + exist in the config (e.g. Azure api_base/api_key) must be fetched through + the router so the deployment credentials are applied. Calling + litellm.aget_responses directly only sees provider env vars, fails, and + leaves the row in "queued" forever. + """ + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="azure", + model_id="deployment-abc", + response_id="resp_upstream_123", + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-router" + mock_job.file_object = {"model": "azure-gpt-5", "id": encoded_response_id} + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[mock_job] + ) + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + + mock_llm_router.aget_responses = AsyncMock( + return_value=ResponsesAPIResponse( + id=encoded_response_id, + object="response", + status="completed", + created_at=int(datetime.now().timestamp()), + output=[], + usage=ResponseAPIUsage( + input_tokens=100, output_tokens=50, total_tokens=150 + ), + ) + ) + + with patch( + "litellm.aget_responses", + new_callable=AsyncMock, + side_effect=AssertionError( + "must not bypass the router for a deployment-scoped response id" + ), + ) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + assert ( + mock_llm_router.aget_responses.call_args[1]["response_id"] + == encoded_response_id + ) + + calls = ( + mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list + ) + assert len(calls) == 1 + assert calls[0][1]["data"]["status"] == "completed" + assert calls[0][1]["where"]["id"]["in"] == ["job-router"] + + @pytest.mark.asyncio + async def test_encrypted_response_id_is_fetched_through_router( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router, monkeypatch + ): + """ + Rows store the *encrypted* response id when responses id security is on. + After decryption the id still carries the deployment model_id, so the + fetch must go through the router (issue #35131). + """ + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.responses.utils import ResponsesAPIRequestUtils + from litellm.types.utils import SpecialEnums + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-for-response-ids") + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", + model_id="deployment-xyz", + response_id="resp_upstream_456", + ) + encrypted_response_id = "resp_" + str( + encrypt_value_helper( + value=SpecialEnums.LITELLM_MANAGED_RESPONSE_API_RESPONSE_ID_COMPLETE_STR.value.format( + encoded_response_id, "test-user", "test-team" + ) + ) + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encrypted_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-encrypted" + mock_job.file_object = {"model": "gpt-5", "id": encrypted_response_id} + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[mock_job] + ) + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + + mock_llm_router.aget_responses = AsyncMock( + return_value=ResponsesAPIResponse( + id=encoded_response_id, + object="response", + status="completed", + created_at=int(datetime.now().timestamp()), + output=[], + usage=None, + ) + ) + + with patch( + "litellm.aget_responses", + new_callable=AsyncMock, + side_effect=AssertionError( + "must not bypass the router for a deployment-scoped response id" + ), + ) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + assert ( + mock_llm_router.aget_responses.call_args[1]["response_id"] + == encoded_response_id + ) + calls = ( + mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list + ) + assert len(calls) == 1 + assert calls[0][1]["where"]["id"]["in"] == ["job-encrypted"] + + @pytest.mark.asyncio + async def test_response_id_without_model_id_uses_sdk( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """Ids that carry no deployment info can't be routed, so fall back to the SDK.""" + mock_job = MagicMock() + mock_job.unified_object_id = "resp_plain_upstream_id" + mock_job.created_by = "test-user" + mock_job.id = "job-plain" + mock_job.file_object = {"model": "gpt-5", "id": "resp_plain_upstream_id"} + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[mock_job] + ) + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + mock_llm_router.aget_responses = AsyncMock( + side_effect=AssertionError("router cannot route an id without a model_id") + ) + + mock_response = ResponsesAPIResponse( + id="resp_plain_upstream_id", + object="response", + status="completed", + created_at=int(datetime.now().timestamp()), + output=[], + usage=None, + ) + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + mock_sdk_aget.return_value = mock_response + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_called_once() + mock_llm_router.aget_responses.assert_not_called() + + @pytest.mark.asyncio + async def test_check_responses_cost_with_incomplete_response( + self, check_responses_cost_instance, mock_prisma_client + ): + """'incomplete' is terminal in the Responses API, so the row must not stay queued.""" + mock_job = MagicMock() + mock_job.unified_object_id = "resp_test_incomplete" + mock_job.created_by = "test-user" + mock_job.id = "job-incomplete" + mock_job.file_object = {"model": "gpt-5", "id": "resp_test_incomplete"} + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[mock_job] + ) + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + + mock_response = ResponsesAPIResponse( + id="resp_incomplete", + object="response", + status="incomplete", + created_at=int(datetime.now().timestamp()), + output=[], + usage=None, + ) + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_aget: + mock_aget.return_value = mock_response + await check_responses_cost_instance.check_responses_cost() + + calls = ( + mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list + ) + assert len(calls) == 1 + assert calls[0][1]["data"]["status"] == "completed" + assert calls[0][1]["where"]["id"]["in"] == ["job-incomplete"] + @pytest.mark.asyncio async def test_check_responses_cost_no_model_in_file_object( self, check_responses_cost_instance, mock_prisma_client From f02e095ddb21063b4d6c1135c59b919c418b4947 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:11:53 +0000 Subject: [PATCH 019/672] fix(cost): stop token-pricing the placeholder input on file content calls --- litellm/litellm_core_utils/litellm_logging.py | 18 ++++-- .../test_litellm_logging.py | 57 +++++++++++++++++++ 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 83d6fcc0bee..aad4ad1f582 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1411,10 +1411,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None) ) - prompt = "" # use for tts cost calc - _input = self.model_call_details.get("input", None) - if _input is not None and isinstance(_input, str): - prompt = _input + prompt = self._prompt_for_cost_calculation() if cache_hit is None: cache_hit = self.model_call_details.get("cache_hit", False) @@ -1473,6 +1470,19 @@ class Logging(LiteLLMLoggingBaseClass): return None + def _prompt_for_cost_calculation(self) -> str: + """ + The raw input string is only priced directly for text-to-speech, which bills per character. + Every other call type gets its billable units from the response usage object, and call types + that carry no usage at all (file content retrieval, and anything else `function_setup` cannot + build messages for) only have the ``"default-message-value"`` placeholder here, so passing the + input along would token-price that placeholder. + """ + if self.call_type not in (CallTypes.speech.value, CallTypes.aspeech.value): + return "" + _input = self.model_call_details.get("input", None) + return _input if isinstance(_input, str) else "" + def _generate_content_result_as_model_response(self, result: object) -> Optional[ModelResponse]: """ Native Google :generateContent bodies report token usage under diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index edc257f4c3f..4a9200aaf7c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -11,6 +11,9 @@ sys.path.insert( import time +import httpx +from openai._legacy_response import HttpxBinaryResponseContent + import litellm from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST from litellm.integrations.custom_logger import CustomLogger @@ -1771,6 +1774,60 @@ def test_response_cost_calculator_does_not_transform_non_generate_content_dict() assert not cost +def _file_content_logging_obj(call_type: str) -> LitellmLogging: + logging_obj = LitellmLogging( + model="gemini-3-flash-preview", + messages="default-message-value", + stream=False, + call_type=call_type, + start_time=time.time(), + litellm_call_id=f"file-content-{call_type}", + function_id=f"file-content-{call_type}", + ) + logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" + logging_obj.model_call_details["input"] = "default-message-value" + logging_obj.optional_params = {} + return logging_obj + + +@pytest.mark.parametrize("call_type", ["afile_content", "file_content"]) +def test_file_content_call_is_not_billed(call_type): + """ + Regression for #35130: file content retrieval has no token usage, but ``function_setup`` + stores the ``"default-message-value"`` placeholder as the logged input, which the cost + calculator then token-priced, billing every call at exactly 3 * input_cost_per_token. + """ + result = HttpxBinaryResponseContent(httpx.Response(status_code=200, content=b"file contents")) + + cost = _file_content_logging_obj(call_type)._response_cost_calculator(result=result) + + assert cost == 0.0 + + +@pytest.mark.parametrize("call_type", ["aspeech", "speech"]) +def test_speech_call_is_still_priced_from_input_characters(call_type): + """tts bills per input character, so speech call types must keep passing the input along.""" + logging_obj = LitellmLogging( + model="tts-1", + messages="the quick brown fox jumped over the lazy dogs", + stream=False, + call_type=call_type, + start_time=time.time(), + litellm_call_id=f"speech-{call_type}", + function_id=f"speech-{call_type}", + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + logging_obj.model_call_details["input"] = "the quick brown fox jumped over the lazy dogs" + logging_obj.optional_params = {} + + result = HttpxBinaryResponseContent(httpx.Response(status_code=200, content=b"audio bytes")) + + cost = logging_obj._response_cost_calculator(result=result) + + assert cost is not None + assert cost > 0 + + def test_sentry_event_scrubber_initialization(monkeypatch): # Step 1: Create a fake sentry_sdk.scrubber module mock_event_scrubber_instance = MagicMock() From b93030f84e7a414d2106528114b09f1fca1ad1aa Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:20:25 +0000 Subject: [PATCH 020/672] fix(vertex_ai): surface real error/status on vertex batch create instead of IndexError 500 --- litellm/llms/vertex_ai/batches/handler.py | 44 ++++++++++++---- .../llms/vertex_ai/batches/transformation.py | 46 ++++++++++++++--- .../llms/vertex_ai/batches/test_handler.py | 50 +++++++++++++++---- .../vertex_ai/batches/test_transformation.py | 43 +++++++++++++++- 4 files changed, 152 insertions(+), 31 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index ada1356fb6b..f0fd5480c75 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -13,7 +13,7 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) -from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.types.llms.openai import CreateBatchRequest from litellm.types.llms.vertex_ai import ( @@ -98,7 +98,9 @@ class VertexAIBatchPrediction(VertexLLM): ) if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) _json_response = response.json() vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( @@ -130,7 +132,9 @@ class VertexAIBatchPrediction(VertexLLM): ) raise if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) _json_response = response.json() vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( @@ -242,7 +246,9 @@ class VertexAIBatchPrediction(VertexLLM): ) if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) _json_response = response.json() vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( @@ -292,7 +298,9 @@ class VertexAIBatchPrediction(VertexLLM): headers=headers, ) if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) _json_response = response.json() vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( @@ -365,7 +373,9 @@ class VertexAIBatchPrediction(VertexLLM): ) if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) _json_response = response.json() vertex_batch_response = ( @@ -390,7 +400,9 @@ class VertexAIBatchPrediction(VertexLLM): params=params, ) if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) _json_response = response.json() vertex_batch_response = ( @@ -475,7 +487,9 @@ class VertexAIBatchPrediction(VertexLLM): raise if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) # HTTPHandler.get() does not accept a timeout parameter retrieve_response = sync_handler.get( @@ -488,7 +502,10 @@ class VertexAIBatchPrediction(VertexLLM): retrieve_response.status_code, retrieve_response.text[:1000], ) - raise Exception(f"Error: {retrieve_response.status_code} {retrieve_response.text}") + raise VertexAIError( + status_code=retrieve_response.status_code, + message=f"Error: {retrieve_response.status_code} {retrieve_response.text}", + ) _json_response = retrieve_response.json() vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( @@ -521,7 +538,9 @@ class VertexAIBatchPrediction(VertexLLM): ) raise if response.status_code != 200: - raise Exception(f"Error: {response.status_code} {response.text}") + raise VertexAIError( + status_code=response.status_code, message=f"Error: {response.status_code} {response.text}" + ) # AsyncHTTPHandler.get() does not accept a timeout parameter retrieve_response = await client.get( @@ -534,7 +553,10 @@ class VertexAIBatchPrediction(VertexLLM): retrieve_response.status_code, retrieve_response.text[:1000], ) - raise Exception(f"Error: {retrieve_response.status_code} {retrieve_response.text}") + raise VertexAIError( + status_code=retrieve_response.status_code, + message=f"Error: {retrieve_response.status_code} {retrieve_response.text}", + ) _json_response = retrieve_response.json() vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response( diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index df903ba7ef0..e4299bcf2a0 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -1,7 +1,9 @@ from typing import Any, Dict, Optional +from urllib.parse import unquote from litellm._uuid import uuid from litellm.llms.vertex_ai.common_utils import ( + VertexAIError, _convert_vertex_datetime_to_openai_datetime, ) from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest @@ -199,16 +201,40 @@ class VertexAIBatchTransformation: gcs_file_uri format: gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/e9412502-2c91-42a6-8e61-f5c294cc0fc8 returns: "publishers/google/models/gemini-1.5-flash-001" + + Raises a 400 `VertexAIError` when the uri carries no parseable model path. """ - from urllib.parse import unquote - - decoded_uri = unquote(gcs_file_uri) - - model_path = decoded_uri.split("publishers/")[1] - parts = model_path.split("/") - model = f"publishers/{'/'.join(parts[:3])}" + model = cls._parse_model_from_gcs_file(gcs_file_uri) + if model is None: + raise VertexAIError( + status_code=400, + message=( + "Vertex AI batch creation requires the model to be part of `input_file_id`, but " + f"'{gcs_file_uri}' contains no 'publishers//models/' path segment. " + "Either upload the input file through LiteLLM (POST /v1/files with " + "custom_llm_provider=vertex_ai), which encodes the model into the returned file id, or " + "pass a uri of the form " + "gs:////publishers//models//" + ), + ) return model + @classmethod + def _parse_model_from_gcs_file(cls, gcs_file_uri: str) -> str | None: + """ + Returns the `publishers//models/` path from a gcs uri, or None if the uri + does not contain one. + """ + _, separator, model_path = unquote(gcs_file_uri).partition("publishers/") + if not separator: + return None + + parts = model_path.split("/") + if len(parts) < 3 or parts[1] != "models" or not parts[2]: + return None + + return f"publishers/{'/'.join(parts[:3])}" + @classmethod def is_unmanaged_gcs_batch_input_file_id(cls, input_file_id: Optional[str]) -> bool: """ @@ -216,7 +242,11 @@ class VertexAIBatchTransformation: LiteLLM-managed unified file id) with a `publishers/` model path that `_get_model_from_gcs_file` can parse. """ - return input_file_id is not None and input_file_id.startswith("gs://") and "publishers/" in input_file_id + return ( + input_file_id is not None + and input_file_id.startswith("gs://") + and cls._parse_model_from_gcs_file(input_file_id) is not None + ) @classmethod def get_bare_model_name_from_gcs_file(cls, gcs_file_uri: str) -> str: diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_handler.py b/tests/test_litellm/llms/vertex_ai/batches/test_handler.py index cacea234777..b9fb5dfe3c5 100644 --- a/tests/test_litellm/llms/vertex_ai/batches/test_handler.py +++ b/tests/test_litellm/llms/vertex_ai/batches/test_handler.py @@ -40,6 +40,7 @@ sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.vertex_ai.batches.handler import ( # noqa: E402 VertexAIBatchPrediction, ) +from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 HMOD = "litellm.llms.vertex_ai.batches.handler" @@ -184,7 +185,7 @@ def test_create_batch_sync_non_200_raises(): client.post.return_value = _http_response(status_code=500) with patch(f"{HMOD}._get_httpx_client", return_value=client): - with pytest.raises(Exception, match="Error: 500"): + with pytest.raises(VertexAIError, match="Error: 500") as exc_info: h.create_batch( _is_async=False, create_batch_data=CREATE_DATA, @@ -196,6 +197,32 @@ def test_create_batch_sync_non_200_raises(): max_retries=None, ) + assert exc_info.value.status_code == 500 + assert "error text" in str(exc_info.value) + + +def test_create_batch_input_file_id_without_model_raises_400_before_post(): + """A gs:// uri with no publishers//models/ path is a 400, not a bare 500.""" + h = _make_handler() + client = MagicMock() + + with patch(f"{HMOD}._get_httpx_client", return_value=client): + with pytest.raises(VertexAIError) as exc_info: + h.create_batch( + _is_async=False, + create_batch_data={"input_file_id": "gs://bucket/batch-input.jsonl"}, + api_base=None, + vertex_credentials=None, + vertex_project=PROJECT, + vertex_location=LOCATION, + timeout=600.0, + max_retries=None, + ) + + assert exc_info.value.status_code == 400 + assert "gs://bucket/batch-input.jsonl" in str(exc_info.value) + client.post.assert_not_called() + def test_create_batch_async_non_200_raises(): h = _make_handler() @@ -216,9 +243,12 @@ def test_create_batch_async_non_200_raises(): timeout=600.0, max_retries=None, ) - with pytest.raises(Exception, match="Error: 403"): + with pytest.raises(VertexAIError, match="Error: 403") as exc_info: _run(coro) + assert exc_info.value.status_code == 403 + assert "error text" in str(exc_info.value) + # =========================================================================== # # retrieve_batch @@ -292,7 +322,7 @@ def test_retrieve_batch_sync_non_200_raises(): patch(f"{HMOD}._get_httpx_client", return_value=MagicMock()), patch(f"{HMOD}.safe_get", return_value=_http_response(status_code=404)), ): - with pytest.raises(Exception, match="Error: 404"): + with pytest.raises(VertexAIError, match="Error: 404"): h.retrieve_batch( _is_async=False, batch_id=BATCH_ID, @@ -438,7 +468,7 @@ def test_list_batches_sync_non_200_raises(): client.get.return_value = _http_response(status_code=500) with patch(f"{HMOD}._get_httpx_client", return_value=client): - with pytest.raises(Exception, match="Error: 500"): + with pytest.raises(VertexAIError, match="Error: 500"): h.list_batches( _is_async=False, after=None, @@ -530,7 +560,7 @@ def test_cancel_batch_sync_cancel_post_non_200_raises(): client.post.return_value = _http_response(status_code=500) with patch(f"{HMOD}._get_httpx_client", return_value=client): - with pytest.raises(Exception, match="Error: 500"): + with pytest.raises(VertexAIError, match="Error: 500"): h.cancel_batch( _is_async=False, batch_id=BATCH_ID, @@ -552,7 +582,7 @@ def test_cancel_batch_sync_retrieve_non_200_raises(): client.get.return_value = _http_response(status_code=404) with patch(f"{HMOD}._get_httpx_client", return_value=client): - with pytest.raises(Exception, match="Error: 404"): + with pytest.raises(VertexAIError, match="Error: 404"): h.cancel_batch( _is_async=False, batch_id=BATCH_ID, @@ -672,7 +702,7 @@ def test_async_retrieve_batch_non_200_raises(): timeout=600.0, max_retries=None, ) - with pytest.raises(Exception, match="Error: 500"): + with pytest.raises(VertexAIError, match="Error: 500"): _run(coro) @@ -726,7 +756,7 @@ def test_async_list_batches_non_200_raises(): timeout=600.0, max_retries=None, ) - with pytest.raises(Exception, match="Error: 500"): + with pytest.raises(VertexAIError, match="Error: 500"): _run(coro) @@ -779,7 +809,7 @@ def test_async_cancel_batch_httpstatuserror_and_retrieve_non_200(): timeout=600.0, max_retries=None, ) - with pytest.raises(Exception, match="Error: 500"): + with pytest.raises(VertexAIError, match="Error: 500"): _run(coro) async_client_post500.get.assert_not_awaited() @@ -801,5 +831,5 @@ def test_async_cancel_batch_httpstatuserror_and_retrieve_non_200(): timeout=600.0, max_retries=None, ) - with pytest.raises(Exception, match="Error: 404"): + with pytest.raises(VertexAIError, match="Error: 404"): _run(coro) diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py b/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py index 1b37ade6b30..8352ec16389 100644 --- a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py @@ -25,6 +25,7 @@ from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402 VertexAIBatchTransformation, ) from litellm.llms.vertex_ai.common_utils import ( # noqa: E402 + VertexAIError, _convert_vertex_datetime_to_openai_datetime, ) from litellm.types.utils import LiteLLMBatch # noqa: E402 @@ -69,6 +70,24 @@ def test_transform_openai_request_missing_input_file_id_raises(): T.transform_openai_batch_request_to_vertex_ai_batch_request({}) +@pytest.mark.parametrize( + "input_file_id", + [ + "gs://bucket/no-model-here.jsonl", + "gs://bucket/publishers/google/gemini-1.5-flash-001/file-uuid", + "gs://bucket/publishers/google/models", + "gs://bucket/publishers/google/models//file-uuid", + ], +) +def test_transform_openai_request_unparseable_model_raises_400(input_file_id: str): + """An input_file_id with no parseable model path is a client error, not an IndexError -> 500.""" + with pytest.raises(VertexAIError) as exc_info: + T.transform_openai_batch_request_to_vertex_ai_batch_request({"input_file_id": input_file_id}) + + assert exc_info.value.status_code == 400 + assert input_file_id in str(exc_info.value) + + # =========================================================================== # # transform_vertex_ai_batch_response_to_openai_batch_response # =========================================================================== # @@ -299,9 +318,29 @@ def test_get_model_from_gcs_file_url_encoded(): assert T._get_model_from_gcs_file(encoded) == "publishers/google/models/gemini-1.5-flash-001" -def test_get_model_from_gcs_file_no_publishers_raises(): - with pytest.raises(IndexError): +def test_get_model_from_gcs_file_no_publishers_raises_400(): + with pytest.raises(VertexAIError) as exc_info: T._get_model_from_gcs_file("gs://bucket/no-model-here.jsonl") + assert exc_info.value.status_code == 400 + + +# =========================================================================== # +# is_unmanaged_gcs_batch_input_file_id +# =========================================================================== # + + +@pytest.mark.parametrize( + "input_file_id, expected", + [ + (INPUT_FILE, True), + (None, False), + ("file-abc123", False), + ("gs://bucket/no-model-here.jsonl", False), + ("gs://bucket/publishers/google/gemini-1.5-flash-001/file-uuid", False), + ], +) +def test_is_unmanaged_gcs_batch_input_file_id(input_file_id, expected): + assert T.is_unmanaged_gcs_batch_input_file_id(input_file_id) is expected # =========================================================================== # From 18d9c7aa21e1308c5ecf05254b29bc0715965bde Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:09:49 +0000 Subject: [PATCH 021/672] fix(bedrock): pass SSE-KMS key through to the batch input-file S3 upload --- .../llms/bedrock/batches/transformation.py | 8 ++- litellm/llms/bedrock/common_utils.py | 19 ++++- litellm/llms/bedrock/files/transformation.py | 16 ++++- litellm/types/router.py | 1 + .../bedrock/batches/test_transformation.py | 2 +- .../test_bedrock_files_transformation.py | 72 ++++++++++++++++++- tests/test_litellm/test_router.py | 31 ++++++++ 7 files changed, 141 insertions(+), 8 deletions(-) diff --git a/litellm/llms/bedrock/batches/transformation.py b/litellm/llms/bedrock/batches/transformation.py index a4ff1c78467..7500531b81a 100644 --- a/litellm/llms/bedrock/batches/transformation.py +++ b/litellm/llms/bedrock/batches/transformation.py @@ -12,7 +12,6 @@ from litellm.litellm_core_utils.cloud_storage_security import ( from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.bedrock import ( BedrockCreateBatchRequest, BedrockCreateBatchResponse, @@ -29,7 +28,7 @@ from litellm.types.llms.openai import ( from litellm.types.utils import LiteLLMBatch, LlmProviders from ..base_aws_llm import BaseAWSLLM -from ..common_utils import CommonBatchFilesUtils +from ..common_utils import CommonBatchFilesUtils, resolve_s3_encryption_key_id # Bedrock batch input files are uploaded as # s3://bucket/litellm-bedrock-files-{model, ":" -> "-"}-{uuid4}.jsonl (see @@ -200,7 +199,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig): ) # Add optional KMS encryption key ID if provided - s3_encryption_key_id = litellm_params.get("s3_encryption_key_id") or get_secret_str("AWS_S3_ENCRYPTION_KEY_ID") + s3_encryption_key_id = resolve_s3_encryption_key_id( + litellm_params=litellm_params, + optional_params=optional_params, + ) if s3_encryption_key_id: s3_output_config["s3EncryptionKeyId"] = s3_encryption_key_id diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 5114677ffc0..9d427fa6f12 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -35,7 +35,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( ) from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.secret_managers.main import get_secret +from litellm.secret_managers.main import get_secret, get_secret_str if TYPE_CHECKING: from litellm.types.llms.openai import AllMessageValues @@ -1313,6 +1313,23 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]: return [] +def resolve_s3_encryption_key_id( + litellm_params: Mapping[str, Any], + optional_params: Mapping[str, Any] | None = None, +) -> str | None: + """ + Resolve the SSE-KMS key configured for Bedrock batch/file S3 objects. + + Precedence: `s3_encryption_key_id` in litellm_params, then optional_params + (client-side / request params), then the AWS_S3_ENCRYPTION_KEY_ID env var. + """ + for source in (litellm_params, optional_params or {}): + value = source.get("s3_encryption_key_id") + if isinstance(value, str) and value: + return value + return get_secret_str("AWS_S3_ENCRYPTION_KEY_ID") + + class CommonBatchFilesUtils: """ Common utilities for Bedrock batch and file operations. diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index d4865a1c87a..d1674b260b4 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -53,7 +53,7 @@ from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums from litellm.utils import get_llm_provider from ..base_aws_llm import BaseAWSLLM -from ..common_utils import BedrockError +from ..common_utils import BedrockError, resolve_s3_encryption_key_id # litellm_params key used to hand the SigV4-signed GET headers from # `transform_file_content_request` to `validate_environment` (the only hook @@ -741,6 +741,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): content=file_content, api_base=api_base, optional_params=optional_params, + s3_encryption_key_id=resolve_s3_encryption_key_id( + litellm_params=litellm_params, + optional_params=optional_params, + ), ) litellm_params["upload_url"] = api_base @@ -758,6 +762,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): content: str, api_base: str, optional_params: dict, + s3_encryption_key_id: str | None = None, ) -> Tuple[dict, str]: """ Sign S3 PUT request using the same proven logic as S3Logger. @@ -790,11 +795,20 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() # Prepare headers with required S3 headers (same as s3_v2.py) + sse_headers = ( + { + "x-amz-server-side-encryption": "aws:kms", + "x-amz-server-side-encryption-aws-kms-key-id": s3_encryption_key_id, + } + if s3_encryption_key_id + else {} + ) request_headers = { "Content-Type": "application/json", # JSONL files are JSON content "x-amz-content-sha256": content_hash, # REQUIRED by S3 "Content-Language": "en", "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0", + **sse_headers, } # Use requests.Request to prepare the request (same pattern as s3_v2.py) diff --git a/litellm/types/router.py b/litellm/types/router.py index 28e4a8272e8..c4d679a2500 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -211,6 +211,7 @@ class CredentialLiteLLMParams(BaseModel): aws_bedrock_runtime_endpoint: Optional[str] = None aws_bedrock_project_id: Optional[str] = None s3_bucket_name: Optional[str] = None + s3_encryption_key_id: Optional[str] = None ## IBM WATSONX ## watsonx_region_name: Optional[str] = None diff --git a/tests/test_litellm/llms/bedrock/batches/test_transformation.py b/tests/test_litellm/llms/bedrock/batches/test_transformation.py index 3681daffe5e..01420eb10df 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_transformation.py +++ b/tests/test_litellm/llms/bedrock/batches/test_transformation.py @@ -172,7 +172,7 @@ def test_create_request_omits_kms_key_when_absent(config): "generate_unique_job_name", return_value="litellm-batch-1", ), patch.object(config.common_utils, "sign_aws_request") as mock_sign, patch( - "litellm.llms.bedrock.batches.transformation.get_secret_str", + "litellm.llms.bedrock.common_utils.get_secret_str", return_value=None, ): mock_sign.return_value = ({}, b"{}") diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index c548fe53e15..a57e5801327 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -442,7 +442,7 @@ class TestBedrockFilesTransformation: captured_optional_params: dict = {} - def fake_sign(content, api_base, optional_params): + def fake_sign(content, api_base, optional_params, s3_encryption_key_id=None): captured_optional_params.update(optional_params) return {"Authorization": "fake"}, content @@ -498,7 +498,7 @@ class TestBedrockFilesTransformation: captured_optional_params: dict = {} - def fake_sign(content, api_base, optional_params): + def fake_sign(content, api_base, optional_params, s3_encryption_key_id=None): captured_optional_params.update(optional_params) return {"Authorization": "fake"}, content @@ -514,6 +514,74 @@ class TestBedrockFilesTransformation: captured_optional_params.get("aws_region_name") == "us-gov-west-1" ), "s3_region_name must override aws_region_name for SigV4 signing" + def _signed_upload_request(self, litellm_params: dict) -> dict: + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + config = BedrockFilesConfig() + jsonl_content = json.dumps( + { + "custom_id": "req-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "bedrock/amazon.nova-pro-v1:0", + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 10, + }, + } + ).encode() + + request = config.transform_create_file_request( + model="amazon.nova-pro-v1:0", + create_file_data={ + "file": ("batch.jsonl", jsonl_content, "application/jsonl"), + "purpose": "batch", + }, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={"s3_bucket_name": "litellm-batch-bucket", **litellm_params}, + ) + assert isinstance(request, dict) + return request + + def test_upload_signs_sse_kms_headers_when_key_configured(self, monkeypatch): + """ + Buckets whose policy requires SSE-KMS reject the batch input-file PutObject + unless the upload carries the aws:kms encryption headers; they must also be + covered by SigV4 SignedHeaders or S3 answers SignatureDoesNotMatch. + """ + monkeypatch.delenv("AWS_S3_ENCRYPTION_KEY_ID", raising=False) + kms_key = "arn:aws:kms:us-west-2:1234:key/abcd" + + request = self._signed_upload_request({"s3_encryption_key_id": kms_key}) + + headers = {key.lower(): value for key, value in request["headers"].items()} + assert headers["x-amz-server-side-encryption"] == "aws:kms" + assert headers["x-amz-server-side-encryption-aws-kms-key-id"] == kms_key + signed_headers = headers["authorization"].split("SignedHeaders=")[1].split(",")[0] + assert "x-amz-server-side-encryption" in signed_headers + assert "x-amz-server-side-encryption-aws-kms-key-id" in signed_headers + + def test_upload_reads_sse_kms_key_from_env(self, monkeypatch): + monkeypatch.setenv("AWS_S3_ENCRYPTION_KEY_ID", "env-kms-key") + + request = self._signed_upload_request({}) + + headers = {key.lower(): value for key, value in request["headers"].items()} + assert headers["x-amz-server-side-encryption-aws-kms-key-id"] == "env-kms-key" + + def test_upload_omits_sse_headers_when_no_key_configured(self, monkeypatch): + monkeypatch.delenv("AWS_S3_ENCRYPTION_KEY_ID", raising=False) + + request = self._signed_upload_request({}) + + headers = {key.lower() for key in request["headers"]} + assert "x-amz-server-side-encryption" not in headers + assert "x-amz-server-side-encryption-aws-kms-key-id" not in headers + def test_openai_passthrough_still_works(self): """ Regression test: ensure OpenAI-compatible models (e.g. gpt-oss) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 46b5ce65c3f..fa047d7ee46 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3666,6 +3666,37 @@ def test_get_deployment_credentials_with_provider_includes_bucket_name(): assert credentials["custom_llm_provider"] == "vertex_ai" +def test_get_deployment_credentials_with_provider_includes_s3_encryption_key_id(): + """ + Regression: s3_encryption_key_id must survive the CredentialLiteLLMParams filter, + otherwise the Bedrock batch input-file upload loses the SSE-KMS key and S3 rejects + the PutObject on buckets whose policy requires aws:kms encryption. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-batch", + "litellm_params": { + "model": "bedrock/anthropic.claude-sonnet-4-20250514-v1:0", + "aws_region_name": "us-west-2", + "s3_bucket_name": "my-batch-bucket", + "s3_encryption_key_id": "arn:aws:kms:us-west-2:1234:key/abcd", + }, + } + ], + ) + + credentials = router.get_deployment_credentials_with_provider( + model_id="bedrock-batch" + ) + + assert credentials is not None + assert ( + credentials["s3_encryption_key_id"] + == "arn:aws:kms:us-west-2:1234:key/abcd" + ) + + def test_get_deployment_credentials_with_provider_resolves_credential_name(): """ Test that get_deployment_credentials_with_provider correctly resolves From 7c56317edf153d61b395f4257476aefdd02f2236 Mon Sep 17 00:00:00 2001 From: Yaroslav Date: Thu, 30 Jul 2026 21:37:28 +0300 Subject: [PATCH 022/672] fix(bedrock): drop toolSpec.strict for Claude Sonnet 5 on Converse (#33196) Bedrock routes Claude Sonnet 5 through the same Anthropic-compatible validator as Opus 4.7/4.8 and Sonnet 4, which rejects toolSpec.strict with 'tools.0.custom.strict: Extra inputs are not permitted'. Set bedrock_converse_supports_strict_tools: false on all six Sonnet 5 entries so the existing gate strips the field, matching the fix shape of #31582 Co-authored-by: Yaroslav Budyanskiy --- ...odel_prices_and_context_window_backup.json | 6 +++++ model_prices_and_context_window.json | 6 +++++ ...edrock_converse_strict_tools_opus_47_48.py | 27 +++++++++++++++---- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e5afc81b641..5e21a868729 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1759,6 +1759,7 @@ "prompt_cache_min_tokens": 2048 }, "anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -1795,6 +1796,7 @@ "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -1831,6 +1833,7 @@ "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, @@ -1867,6 +1870,7 @@ "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, @@ -1903,6 +1907,7 @@ "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, @@ -1939,6 +1944,7 @@ "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5cf99ba8bac..7587f71bffc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1759,6 +1759,7 @@ "prompt_cache_min_tokens": 2048 }, "anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -1795,6 +1796,7 @@ "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -1831,6 +1833,7 @@ "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, @@ -1867,6 +1870,7 @@ "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, @@ -1903,6 +1907,7 @@ "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, @@ -1939,6 +1944,7 @@ "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_1hr": 4.4e-06, "cache_read_input_token_cost": 2.2e-07, diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py index 791982fc3dc..b02324af0a5 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py @@ -1,9 +1,9 @@ """Regression tests for Bedrock Converse ``toolSpec.strict`` forwarding. -Bedrock Converse routes Claude Opus 4.7/4.8 and Claude Sonnet 4 through an -Anthropic-compatible validator that rejects ``toolSpec.strict`` even though -Anthropic's native API accepts ``strict`` as a top-level tool field. See -BerriAI/litellm#31582. +Bedrock Converse routes Claude Opus 4.7/4.8, Claude Sonnet 4 and Claude +Sonnet 5 through an Anthropic-compatible validator that rejects +``toolSpec.strict`` even though Anthropic's native API accepts ``strict`` +as a top-level tool field. See BerriAI/litellm#31582. """ import pytest @@ -48,12 +48,18 @@ _STRICT_TOOL = [ "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", "bedrock/eu.anthropic.claude-sonnet-4-20250514-v1:0", "bedrock/apac.anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-sonnet-5", + "bedrock/global.anthropic.claude-sonnet-5", + "bedrock/us.anthropic.claude-sonnet-5", + "bedrock/eu.anthropic.claude-sonnet-5", + "bedrock/au.anthropic.claude-sonnet-5", + "bedrock/jp.anthropic.claude-sonnet-5", ], ) def test_bedrock_tools_pt_strict_dropped_for_strict_unsupported_models( model_id: str, ) -> None: - """Opus 4.7/4.8 and Sonnet 4 reject toolSpec.strict and additionalProperties.""" + """Opus 4.7/4.8, Sonnet 4 and Sonnet 5 reject toolSpec.strict and additionalProperties.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) tool_spec = result[0]["toolSpec"] assert ( @@ -129,6 +135,11 @@ def test_bedrock_converse_supports_strict_tools_helper() -> None: ) is False ) + assert bedrock_converse_supports_strict_tools("anthropic.claude-sonnet-5") is False + assert ( + bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-sonnet-5") + is False + ) @pytest.mark.parametrize( @@ -143,6 +154,12 @@ def test_bedrock_converse_supports_strict_tools_helper() -> None: "us.anthropic.claude-sonnet-4-20250514-v1:0", "eu.anthropic.claude-sonnet-4-20250514-v1:0", "apac.anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-sonnet-5", + "global.anthropic.claude-sonnet-5", + "us.anthropic.claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + "au.anthropic.claude-sonnet-5", + "jp.anthropic.claude-sonnet-5", ], ) def test_strict_tools_flag_set_in_model_cost_map(cost_map_key: str) -> None: From 3e4669dbc5d31a261e65af8e021d2ad12a2d15c4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:03:19 +0000 Subject: [PATCH 023/672] fix(cost): track OpenAI/Azure web search tool cost per call Adds search_context_cost_per_query pricing for the 82 OpenAI/Azure models that advertise supports_web_search but had none (gpt-5 family, o-series, deep-research at $0.01/call; gpt-4.1 at $0.025/call), so built-in web search is no longer billed as $0. Also counts web_search_call items in Responses output so N searches bill N times instead of once; usage-count providers (gemini, anthropic, xai, vertex) still route through get_cost_for_web_search_request and are unaffected. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 20 +- ...odel_prices_and_context_window_backup.json | 410 ++++++++++++++++++ model_prices_and_context_window.json | 410 ++++++++++++++++++ .../test_tool_call_cost_tracking.py | 85 ++++ 4 files changed, 924 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 221b1ae6eab..1d8f2a6c965 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -117,10 +117,28 @@ class StandardBuiltInToolCostTracking: if result is not None: return result - return StandardBuiltInToolCostTracking.get_cost_for_web_search( + per_call_cost = StandardBuiltInToolCostTracking.get_cost_for_web_search( web_search_options=standard_built_in_tools_params.get("web_search_options", None), model_info=model_info, ) + return per_call_cost * StandardBuiltInToolCostTracking._count_web_search_calls(response_object) + + @staticmethod + def _count_web_search_calls(response_object: object) -> int: + """ + Number of web searches to bill for on the per-call pricing path. + + Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by + get_cost_for_web_search_request and never reach here. This path prices per call, so it must count + the web_search_call items. Chat-completions responses only expose url_citation annotations with no + count, so they floor to a single billable search. + """ + if isinstance(response_object, ResponsesAPIResponse): + count = sum( + 1 for output_item in response_object.output if getattr(output_item, "type", None) == "web_search_call" + ) + return max(count, 1) + return 1 @staticmethod def _handle_file_search_cost( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5e21a868729..193d2bc2025 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5760,6 +5760,11 @@ "supports_vision": true }, "azure/gpt-5.2-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -5791,6 +5796,11 @@ "supports_web_search": true }, "azure/gpt-5.2-pro-2025-12-11": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -6044,6 +6054,11 @@ "supports_vision": true }, "azure/gpt-5.4-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6079,6 +6094,11 @@ "supports_web_search": true }, "azure/gpt-5.4-pro-2026-03-05": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6114,6 +6134,11 @@ "supports_web_search": true }, "azure/gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6159,6 +6184,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6204,6 +6234,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -6249,6 +6284,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_above_272k_tokens": 2e-07, "cache_read_input_token_cost_priority": 2e-07, @@ -6294,6 +6334,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6336,6 +6381,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6378,6 +6428,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 6.875e-07, @@ -6420,6 +6475,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, "cache_read_input_token_cost_priority": 2.75e-07, @@ -6462,6 +6522,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6504,6 +6569,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6546,6 +6616,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 6.875e-07, @@ -6588,6 +6663,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, "cache_read_input_token_cost_priority": 2.75e-07, @@ -6630,6 +6710,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6675,6 +6760,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6717,6 +6807,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6759,6 +6854,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6801,6 +6901,11 @@ "supports_web_search": true }, "azure/us/gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6840,6 +6945,11 @@ "supports_web_search": true }, "azure/eu/gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6879,6 +6989,11 @@ "supports_web_search": true }, "azure/gpt-5.5-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6918,6 +7033,11 @@ "supports_low_reasoning_effort": false }, "azure/gpt-5.5-pro-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6953,6 +7073,11 @@ "supports_web_search": true }, "azure/gpt-5.4-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -6988,6 +7113,11 @@ "supports_xhigh_reasoning_effort": false }, "azure/gpt-5.4-mini-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -7023,6 +7153,11 @@ "supports_xhigh_reasoning_effort": false }, "azure/gpt-5.4-nano": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7058,6 +7193,11 @@ "supports_xhigh_reasoning_effort": false }, "azure/gpt-5.4-nano-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7521,6 +7661,11 @@ "supports_vision": true }, "azure/o3-deep-research": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -21452,6 +21597,11 @@ "supports_tool_choice": true }, "gpt-4.1": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, @@ -21489,6 +21639,11 @@ "supports_web_search": true }, "gpt-4.1-2025-04-14": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -21523,6 +21678,11 @@ "supports_web_search": true }, "gpt-4.1-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, @@ -21560,6 +21720,11 @@ "supports_web_search": true }, "gpt-4.1-mini-2025-04-14": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -22706,6 +22871,11 @@ "supports_pdf_input": true }, "gpt-5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -22748,6 +22918,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -22787,6 +22962,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-2025-11-13": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -22826,6 +23006,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-chat-latest": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -22865,6 +23050,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -22905,6 +23095,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-2025-12-11": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -22945,6 +23140,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-chat-latest": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -22983,6 +23183,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.3-chat-latest": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -23021,6 +23226,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -23055,6 +23265,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro-2025-12-11": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -23089,6 +23304,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, "cache_creation_input_token_cost_flex": 3.125e-06, @@ -23142,6 +23362,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, "cache_creation_input_token_cost_flex": 3.125e-06, @@ -23195,6 +23420,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 3.125e-06, "cache_creation_input_token_cost_above_272k_tokens": 6.25e-06, "cache_creation_input_token_cost_flex": 1.5625e-06, @@ -23248,6 +23478,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, "cache_creation_input_token_cost_flex": 6.25e-07, @@ -23301,6 +23536,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, @@ -23350,6 +23590,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, @@ -23399,6 +23644,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.5-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23444,6 +23694,11 @@ "supports_low_reasoning_effort": false }, "gpt-5.5-pro-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23582,6 +23837,11 @@ "supports_vision": true }, "gpt-5.4-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23626,6 +23886,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.4-pro-2026-03-05": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23670,6 +23935,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.4-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, @@ -23716,6 +23986,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.4-mini-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, @@ -23762,6 +24037,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.4-nano": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, @@ -23805,6 +24085,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.4-nano-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, @@ -23848,6 +24133,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -23884,6 +24174,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-pro-2025-10-06": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -23920,6 +24215,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-2025-08-07": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -24032,6 +24332,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", @@ -24066,6 +24371,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -24103,6 +24413,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-max": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", @@ -24137,6 +24452,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, @@ -24174,6 +24494,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -24211,6 +24536,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.3-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -24248,6 +24578,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -24290,6 +24625,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-mini-2025-08-07": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -24332,6 +24672,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-nano": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, @@ -24372,6 +24717,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-nano-2025-08-07": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, @@ -28548,6 +28898,11 @@ "supports_vision": true }, "o3": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -28586,6 +28941,11 @@ "supports_web_search": true }, "o3-2025-04-16": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "openai", @@ -28618,6 +28978,11 @@ "supports_web_search": true }, "o3-deep-research": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_batches": 5e-06, @@ -28652,6 +29017,11 @@ "supports_web_search": true }, "o3-deep-research-2025-06-26": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_batches": 5e-06, @@ -28720,6 +29090,11 @@ "supports_vision": false }, "o3-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -28751,6 +29126,11 @@ "supports_web_search": true }, "o3-pro-2025-06-10": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -28782,6 +29162,11 @@ "supports_web_search": true }, "o4-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_flex": 1.375e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -28807,6 +29192,11 @@ "supports_web_search": true }, "o4-mini-2025-04-16": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "openai", @@ -28826,6 +29216,11 @@ "supports_web_search": true }, "o4-mini-deep-research": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -28860,6 +29255,11 @@ "supports_web_search": true }, "o4-mini-deep-research-2025-06-26": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -43526,6 +43926,11 @@ ] }, "gpt-5-search-api": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", @@ -43548,6 +43953,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-search-api-2025-10-14": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7587f71bffc..35bf17675cb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5760,6 +5760,11 @@ "supports_vision": true }, "azure/gpt-5.2-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -5791,6 +5796,11 @@ "supports_web_search": true }, "azure/gpt-5.2-pro-2025-12-11": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -6044,6 +6054,11 @@ "supports_vision": true }, "azure/gpt-5.4-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6079,6 +6094,11 @@ "supports_web_search": true }, "azure/gpt-5.4-pro-2026-03-05": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6114,6 +6134,11 @@ "supports_web_search": true }, "azure/gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6159,6 +6184,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6204,6 +6234,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -6249,6 +6284,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_above_272k_tokens": 2e-07, "cache_read_input_token_cost_priority": 2e-07, @@ -6294,6 +6334,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6336,6 +6381,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6378,6 +6428,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 6.875e-07, @@ -6420,6 +6475,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, "cache_read_input_token_cost_priority": 2.75e-07, @@ -6462,6 +6522,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6504,6 +6569,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.375e-06, @@ -6546,6 +6616,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 6.875e-07, @@ -6588,6 +6663,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.1e-07, "cache_read_input_token_cost_above_272k_tokens": 2.2e-07, "cache_read_input_token_cost_priority": 2.75e-07, @@ -6630,6 +6710,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6675,6 +6760,11 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6717,6 +6807,11 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6759,6 +6854,11 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -6801,6 +6901,11 @@ "supports_web_search": true }, "azure/us/gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6840,6 +6945,11 @@ "supports_web_search": true }, "azure/eu/gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -6879,6 +6989,11 @@ "supports_web_search": true }, "azure/gpt-5.5-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6918,6 +7033,11 @@ "supports_low_reasoning_effort": false }, "azure/gpt-5.5-pro-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -6953,6 +7073,11 @@ "supports_web_search": true }, "azure/gpt-5.4-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -6988,6 +7113,11 @@ "supports_xhigh_reasoning_effort": false }, "azure/gpt-5.4-mini-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -7023,6 +7153,11 @@ "supports_xhigh_reasoning_effort": false }, "azure/gpt-5.4-nano": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7058,6 +7193,11 @@ "supports_xhigh_reasoning_effort": false }, "azure/gpt-5.4-nano-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7521,6 +7661,11 @@ "supports_vision": true }, "azure/o3-deep-research": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -21527,6 +21672,11 @@ "supports_tool_choice": true }, "gpt-4.1": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, @@ -21564,6 +21714,11 @@ "supports_web_search": true }, "gpt-4.1-2025-04-14": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -21598,6 +21753,11 @@ "supports_web_search": true }, "gpt-4.1-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, @@ -21635,6 +21795,11 @@ "supports_web_search": true }, "gpt-4.1-mini-2025-04-14": { + "search_context_cost_per_query": { + "search_context_size_high": 0.025, + "search_context_size_low": 0.025, + "search_context_size_medium": 0.025 + }, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -22781,6 +22946,11 @@ "supports_pdf_input": true }, "gpt-5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -22823,6 +22993,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -22862,6 +23037,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-2025-11-13": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -22901,6 +23081,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-chat-latest": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -22940,6 +23125,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -22980,6 +23170,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-2025-12-11": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -23020,6 +23215,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-chat-latest": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -23058,6 +23258,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.3-chat-latest": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -23096,6 +23301,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -23130,6 +23340,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-pro-2025-12-11": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2.1e-05, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -23164,6 +23379,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.6": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, "cache_creation_input_token_cost_flex": 3.125e-06, @@ -23217,6 +23437,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-sol": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, "cache_creation_input_token_cost_flex": 3.125e-06, @@ -23270,6 +23495,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-terra": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 3.125e-06, "cache_creation_input_token_cost_above_272k_tokens": 6.25e-06, "cache_creation_input_token_cost_flex": 1.5625e-06, @@ -23323,6 +23553,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-luna": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, "cache_creation_input_token_cost_flex": 6.25e-07, @@ -23376,6 +23611,11 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.5": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, @@ -23425,6 +23665,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.5-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_flex": 2.5e-07, @@ -23474,6 +23719,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.5-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23519,6 +23769,11 @@ "supports_low_reasoning_effort": false }, "gpt-5.5-pro-2026-04-23": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23657,6 +23912,11 @@ "supports_vision": true }, "gpt-5.4-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23701,6 +23961,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.4-pro-2026-03-05": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -23745,6 +24010,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.4-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, @@ -23791,6 +24061,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.4-mini-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, @@ -23837,6 +24112,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.4-nano": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, @@ -23880,6 +24160,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5.4-nano-2026-03-17": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, @@ -23923,6 +24208,11 @@ "supports_minimal_reasoning_effort": false }, "gpt-5-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -23959,6 +24249,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-pro-2025-10-06": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -23995,6 +24290,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-2025-08-07": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, @@ -24107,6 +24407,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", @@ -24141,6 +24446,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -24178,6 +24488,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-max": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", @@ -24212,6 +24527,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.1-codex-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, @@ -24249,6 +24569,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.2-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -24286,6 +24611,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.3-codex": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -24323,6 +24653,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -24365,6 +24700,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-mini-2025-08-07": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -24407,6 +24747,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-nano": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, @@ -24447,6 +24792,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-nano-2025-08-07": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, @@ -28623,6 +28973,11 @@ "supports_vision": true }, "o3": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -28661,6 +29016,11 @@ "supports_web_search": true }, "o3-2025-04-16": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "openai", @@ -28693,6 +29053,11 @@ "supports_web_search": true }, "o3-deep-research": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_batches": 5e-06, @@ -28727,6 +29092,11 @@ "supports_web_search": true }, "o3-deep-research-2025-06-26": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_batches": 5e-06, @@ -28795,6 +29165,11 @@ "supports_vision": false }, "o3-pro": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -28826,6 +29201,11 @@ "supports_web_search": true }, "o3-pro-2025-06-10": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -28857,6 +29237,11 @@ "supports_web_search": true }, "o4-mini": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_flex": 1.375e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -28882,6 +29267,11 @@ "supports_web_search": true }, "o4-mini-2025-04-16": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "openai", @@ -28901,6 +29291,11 @@ "supports_web_search": true }, "o4-mini-deep-research": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -28935,6 +29330,11 @@ "supports_web_search": true }, "o4-mini-deep-research-2025-06-26": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -43647,6 +44047,11 @@ ] }, "gpt-5-search-api": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", @@ -43669,6 +44074,11 @@ "supports_minimal_reasoning_effort": true }, "gpt-5-search-api-2025-10-14": { + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 24fd3c94ee3..f194db7676a 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -602,5 +602,90 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model( ) +def _openai_responses_with_web_search_calls(model, num_calls): + from litellm.types.llms.openai import ResponsesAPIResponse + from openai.types.responses.response_function_web_search import ( + ActionSearch, + ResponseFunctionWebSearch, + ) + + output = [ + ResponseFunctionWebSearch( + id=f"ws_{i}", + type="web_search_call", + status="completed", + action=ActionSearch(type="search", query="latest news"), + ) + for i in range(num_calls) + ] + return ResponsesAPIResponse( + id="resp_1", + created_at=0, + model=model, + object="response", + output=output, + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + +def test_openai_responses_web_search_priced_per_call(local_model_cost_map): + """ + Regression for LIT-5013 bug 1: OpenAI reasoning models (gpt-5 family, o-series, deep-research) + carry supports_web_search but had no search_context_cost_per_query, so get_cost_for_web_search_request + (no openai branch) returned None and the default fallback billed web search as $0. gpt-5-nano now + prices at $0.01 per call, and two web_search_call items in the Responses output must bill 2 x $0.01. + """ + from litellm.types.utils import Usage + + model = "gpt-5-nano" + per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] + assert per_call == 0.01 + + response = _openai_responses_with_web_search_calls(model, num_calls=2) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(2 * per_call), ( + f"gpt-5-nano web search must bill 2 x ${per_call}, got ${cost}" + ) + + +def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_map): + """ + Regression for LIT-5013 bug 2: web_search_call detection was binary, so a Responses output with + multiple web searches was charged once. gpt-4o-search-preview carries per-call pricing; N calls + must bill N times, and a single call must still bill exactly once. + """ + from litellm.types.utils import Usage + + model = "gpt-4o-search-preview" + per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] + usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + + for num_calls in (1, 3): + response = _openai_responses_with_web_search_calls(model, num_calls=num_calls) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=usage, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + assert cost == pytest.approx(num_calls * per_call), ( + f"{num_calls} web searches must bill {num_calls} x ${per_call}, got ${cost}" + ) + + # Note: File search integration test removed due to complex annotation detection logic # The unit tests in test_azure_assistant_cost_tracking.py provide comprehensive coverage From 0b809cf7d68e368b7a7ee90d7134c4841c944e3c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 03:21:20 +0000 Subject: [PATCH 024/672] fix(anthropic adapter): stop indexing choices[0] on choiceless streaming chunks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../adapters/streaming_iterator.py | 30 +++++++ .../test_streaming_iterator_empty_choices.py | 87 +++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index d9bcfa19a7f..194cbcc9327 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -329,6 +329,26 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): merged_chunk["context_management"] = ContextManagementResponse(applied_edits=list(self.applied_edits)) return self._augment_message_delta_usage(merged_chunk) + def _handle_choiceless_chunk(self, chunk: Any) -> bool: + """Consume an OpenAI-compatible chunk that carries no ``choices``. + + ``choices`` is legitimately empty on metadata-only chunks; the final + usage chunk emitted when ``stream_options.include_usage`` is set is the + common case (vLLM and other OpenAI-compatible servers do this). Such a + chunk carries no content-block information, so the caller must not run + the content-block state machine over it. + + Returns True when a merged ``message_delta`` was queued (usage folded + into the held stop-reason chunk); False when the chunk should be + skipped entirely. + """ + if self.holding_stop_reason_chunk is not None and getattr(chunk, "usage", None) is not None: + self.chunk_queue.append(self._merge_usage_into_held_stop_reason_chunk(chunk)) + self.queued_usage_chunk = True + self.holding_stop_reason_chunk = None + return True + return False + def _ensure_context_management_attached(self, message_delta_chunk: Dict[str, Any]) -> Dict[str, Any]: """Attach ``context_management`` to a ``message_delta`` chunk if ``self.applied_edits`` is non-empty and the chunk does not already @@ -490,6 +510,11 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): if chunk == "None" or chunk is None: raise Exception + if not getattr(chunk, "choices", None): + if self._handle_choiceless_chunk(chunk): + return self.chunk_queue.popleft() + continue + should_start_new_block = self._should_start_new_content_block(chunk) is_opening_first_block = self.sent_content_block_start is False if is_opening_first_block and self._is_blank_delta(chunk): @@ -713,6 +738,11 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): if chunk == "None" or chunk is None: raise Exception + if not getattr(chunk, "choices", None): + if self._handle_choiceless_chunk(chunk): + return self.chunk_queue.popleft() + continue + should_start_new_block = self._should_start_new_content_block(chunk) is_opening_first_block = self.sent_content_block_start is False if is_opening_first_block and self._is_blank_delta(chunk): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py new file mode 100644 index 00000000000..3e85872f1e5 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py @@ -0,0 +1,87 @@ +""" +Regression tests for OpenAI-compatible chunks with an empty ``choices`` list. + +``choices: []`` is valid OpenAI-compatible streaming: vLLM (and OpenAI itself, +when ``stream_options.include_usage`` is set) emits a final usage chunk with no +choices, and some gateways emit metadata-only chunks mid-stream. The adapter +used to index ``chunk.choices[0]`` unconditionally, so such a chunk raised +``IndexError: list index out of range`` and killed the ``/v1/messages`` stream. +""" + +import asyncio +import json +from typing import Any, AsyncIterator, Dict, List, Optional + +from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( + AnthropicStreamWrapper, +) +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage + + +def _text_chunk(text: str) -> ModelResponseStream: + return ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=text), finish_reason=None)] + ) + + +def _finish_chunk() -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]) + + +def _empty_choices_chunk(usage: Optional[Usage] = None) -> ModelResponseStream: + return ModelResponseStream(choices=[], usage=usage) + + +def _collect_async(wrapper: AnthropicStreamWrapper) -> str: + async def _run() -> str: + return "".join( + [raw.decode() if isinstance(raw, bytes) else raw async for raw in wrapper.async_anthropic_sse_wrapper()] + ) + + return asyncio.run(_run()) + + +def _message_delta(sse: str) -> Dict[str, Any]: + return next( + json.loads(line[len("data: ") :]) + for block in sse.split("\n\n") + for line in block.splitlines() + if line.startswith("data: ") and '"message_delta"' in line + ) + + +def test_leading_metadata_chunk_without_choices_does_not_kill_stream(): + """A metadata-only chunk before any content must be skipped, not indexed.""" + chunks: List[ModelResponseStream] = [ + _empty_choices_chunk(), + _text_chunk("Hello"), + _text_chunk(" there"), + _finish_chunk(), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="mock-model") + events = list(wrapper) + + text = "".join( + event["delta"]["text"] for event in events if event.get("type") == "content_block_delta" + ) + assert text == "Hello there" + assert events[-1]["type"] == "message_stop" + + +def test_final_usage_chunk_without_choices_is_merged_into_message_delta(): + """The vLLM/OpenAI final usage chunk carries no choices; its usage must + still land on the Anthropic ``message_delta``.""" + usage = Usage(prompt_tokens=10, completion_tokens=3, total_tokens=13) + + async def _aiter() -> "AsyncIterator[ModelResponseStream]": + for chunk in [_text_chunk("Hi"), _finish_chunk(), _empty_choices_chunk(usage)]: + yield chunk + + sse = _collect_async(AnthropicStreamWrapper(completion_stream=_aiter(), model="mock-model")) + + message_delta = _message_delta(sse) + assert message_delta["delta"]["stop_reason"] == "end_turn" + assert message_delta["usage"]["input_tokens"] == 10 + assert message_delta["usage"]["output_tokens"] == 3 + assert "Hi" in sse + assert "message_stop" in sse From f0ffc6507e1d21daa4f3a13a0245daa55effccd2 Mon Sep 17 00:00:00 2001 From: Rithvik Mysore Suresh Date: Fri, 31 Jul 2026 10:16:35 -0400 Subject: [PATCH 025/672] fix(batches): keep managed files on owner Managed files and batches are provider-owned. Cross-model fallbacks can dispatch creation with credentials that cannot access the input file and replace the owning provider's validation error.\n\nCloses #35359 --- litellm/proxy/batches_endpoints/endpoints.py | 5 ++- .../proxy/batches_endpoints/test_endpoints.py | 3 +- tests/test_litellm/test_router.py | 45 +++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index a91b29002e3..b7713d388ea 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -262,7 +262,10 @@ async def create_batch( detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - response = await llm_router.acreate_batch(**_create_batch_data) + response = await llm_router.acreate_batch( + **_create_batch_data, + disable_fallbacks=True, + ) response.input_file_id = input_file_id response._hidden_params["unified_file_id"] = unified_file_id else: diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index 6a185988c9b..b382313ea1f 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -469,7 +469,7 @@ async def test_create__fallback_body_custom_llm_provider(harness): @pytest.mark.asyncio -async def test_create__unified_file_id_single_model(harness): +async def test_create__unified_file_id_single_model_disables_cross_model_fallbacks(harness): set_body( harness, { @@ -489,6 +489,7 @@ async def test_create__unified_file_id_single_model(harness): harness.litellm_acreate.assert_not_called() # model injected from the unified id, input_file_id restored, hidden param set assert harness.router_kwargs()["model"] == "gpt-4o-mini" + assert harness.router_kwargs()["disable_fallbacks"] is True assert resp.input_file_id == "litellm_proxy_unified_id" assert resp._hidden_params["unified_file_id"] == "unified-xyz" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 46b5ce65c3f..aa917757bbf 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6054,6 +6054,51 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +@pytest.mark.asyncio +async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error(): + router = litellm.Router( + model_list=[ + { + "model_name": "owning-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-owning", + }, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "azure/gpt-4o-mini", + "api_key": "sk-fallback", + "api_base": "https://fallback.openai.azure.com", + "api_version": "2024-08-01-preview", + }, + }, + ], + fallbacks=[{"owning-model": ["fallback-model"]}], + num_retries=0, + ) + owning_provider_error = litellm.BadRequestError( + message="completion_window must be one of: 24h", + model="openai/gpt-4o-mini", + llm_provider="openai", + ) + mock_create = AsyncMock(side_effect=owning_provider_error) + + with patch.object(router, "_acreate_batch", mock_create): + with pytest.raises(litellm.BadRequestError, match="24h"): + await router.acreate_batch( + model="owning-model", + input_file_id="file-owned-by-openai", + endpoint="/v1/chat/completions", + completion_window="5m", + disable_fallbacks=True, + ) + + mock_create.assert_awaited_once() + assert mock_create.call_args.kwargs["model"] == "owning-model" + + @pytest.mark.asyncio async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): import httpx From 55726fc09e979fee39680a465b1e04b95def8c3b Mon Sep 17 00:00:00 2001 From: Rithvik Mysore Suresh Date: Fri, 31 Jul 2026 10:28:52 -0400 Subject: [PATCH 026/672] fix(batches): override existing fallback flag Build one kwargs mapping so managed-file ownership always disables cross-model fallback without duplicating a request-enriched key. --- litellm/proxy/batches_endpoints/endpoints.py | 3 +-- tests/test_litellm/proxy/batches_endpoints/test_endpoints.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index b7713d388ea..a5a03320f7a 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -263,8 +263,7 @@ async def create_batch( ) response = await llm_router.acreate_batch( - **_create_batch_data, - disable_fallbacks=True, + **{**_create_batch_data, "disable_fallbacks": True}, ) response.input_file_id = input_file_id response._hidden_params["unified_file_id"] = unified_file_id diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index b382313ea1f..f8bc3e10d79 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -476,6 +476,7 @@ async def test_create__unified_file_id_single_model_disables_cross_model_fallbac "input_file_id": "litellm_proxy_unified_id", "endpoint": "/v1/chat/completions", "completion_window": "24h", + "disable_fallbacks": False, }, ) with ( From efb5f74173879660d4a79eed66d882da57980946 Mon Sep 17 00:00:00 2001 From: Rithvik Mysore Suresh Date: Fri, 31 Jul 2026 10:39:05 -0400 Subject: [PATCH 027/672] fix(batches): overwrite fallback flag in place Avoid a fresh mutable kwargs mapping while still replacing any request-enriched value before router dispatch. --- litellm/proxy/batches_endpoints/endpoints.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index a5a03320f7a..f94518b16b6 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -262,9 +262,8 @@ async def create_batch( detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - response = await llm_router.acreate_batch( - **{**_create_batch_data, "disable_fallbacks": True}, - ) + _create_batch_data.update(disable_fallbacks=True) # pyright: ignore[reportCallIssue] # router flag + response = await llm_router.acreate_batch(**_create_batch_data) response.input_file_id = input_file_id response._hidden_params["unified_file_id"] = unified_file_id else: From bae58eb4e0aa015d5085264e8b0d9a342f163ce5 Mon Sep 17 00:00:00 2001 From: eugene-yao-zocdoc Date: Fri, 31 Jul 2026 13:03:13 -0400 Subject: [PATCH 028/672] fix(anthropic): preserve mid-turn system messages Generated with AI Co-Authored-By: Claude Code --- .../chat/guardrail_translation/handler.py | 209 ++++-- .../adapters/transformation.py | 48 +- .../responses_adapters/transformation.py | 49 +- litellm/types/guardrails.py | 47 +- litellm/types/llms/anthropic.py | 12 + .../test_anthropic_guardrail_handler.py | 699 ++++++++++++++++++ ...al_pass_through_adapters_transformation.py | 218 ++++++ .../context_management/test_compact.py | 54 ++ .../test_responses_adapters_transformation.py | 138 ++++ 9 files changed, 1380 insertions(+), 94 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index a549db94224..82707e741c0 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -13,6 +13,7 @@ Pattern Overview: """ import json +from copy import deepcopy from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast from litellm._logging import verbose_proxy_logger @@ -24,7 +25,6 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTra from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, - openai_messages_without_system, openai_messages_without_tool, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( @@ -59,14 +59,10 @@ if TYPE_CHECKING: class AnthropicMessagesHandler(BaseTranslation): - """ - Handler for processing Anthropic messages with guardrails. + """Process Anthropic messages with guardrails. - This class provides methods to: - 1. Process input messages (pre-call hook) - 2. Process output responses (post-call hook) - - Methods can be overridden to customize behavior for different message formats. + In-sequence system entries are untrusted client input. This handler scans and preserves + them through guardrail rewrites; downstream provider handling is out of scope. """ def __init__(self): @@ -279,14 +275,26 @@ class AnthropicMessagesHandler(BaseTranslation): skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) - chat_completion_compatible_request = self._translate_to_openai(data) + # Exclude only the trusted top-level prompt. In-sequence system entries are untrusted + # and must stay aligned with texts_to_check for positional masking. When the top-level + # prompt is included, the pre-existing count mismatch disables positional masking. + translation_source = { # mutable-ok: API message payload + key: value for key, value in data.items() if key != "system" + } # mutable-ok: API message payload + chat_completion_compatible_request = self._translate_to_openai(translation_source) structured_messages = cast( List[AllMessageValues], chat_completion_compatible_request.get("messages", []), ) - if skip_system: - structured_messages = openai_messages_without_system(structured_messages) + has_midturn_system_message = any( + str(message.get("role") or "").lower() == "system" for message in structured_messages + ) + hoisted_system_message: AllMessageValues | None = None + if not skip_system: + hoisted_system_message = self._hoisted_top_level_system_message(data) + if hoisted_system_message is not None: + structured_messages.insert(0, hoisted_system_message) if skip_tool: structured_messages = openai_messages_without_tool(structured_messages) @@ -346,7 +354,12 @@ class AnthropicMessagesHandler(BaseTranslation): guardrailed_structured_messages is not None and guardrailed_structured_messages is not original_structured_messages ): - self._write_back_structured_messages(data, guardrailed_structured_messages) + self._write_back_structured_messages( + data, + guardrailed_structured_messages, + hoisted_system_message=hoisted_system_message, + preserve_system_messages=has_midturn_system_message, + ) else: # Step 3: Map guardrail responses back to original message structure await self._apply_guardrail_responses_to_input( @@ -359,36 +372,120 @@ class AnthropicMessagesHandler(BaseTranslation): return data - @staticmethod - def _write_back_structured_messages(data: dict, structured_messages: list) -> None: - """Convert compressed structured_messages back to Anthropic format and write to data. + def _hoisted_top_level_system_message( + self, data: dict + ) -> AllMessageValues | None: # mutable-ok: API message payload + """Return the system message produced by translating the top-level prompt.""" + system = data.get("system") + if not system: + return None + probe = self._translate_to_openai( + { # mutable-ok: API message payload + "model": data.get("model") or "", + "messages": [], # mutable-ok: API message payload + "system": system, + } + ) + hoisted = probe.get("messages") or [] # mutable-ok: API message payload + return hoisted[0] if hoisted else None - ``anthropic_messages_pt`` merges every run of consecutive user/tool rows - into a single message, so a turn carrying only tool results and the user - turn that follows it come back fused, and the request the model sees no - longer has the boundaries the client sent. Converting a row at a time - would keep them apart but breaks tool pairing: an assistant row whose - tool results sit outside its own call reads as an orphaned tool call, - and under ``modify_params`` the sanitizer answers it with a synthetic - "tool execution skipped" result and drops the real one. Converting each - assistant row together with the tool rows that answer it, and every - other row on its own, satisfies both. - """ + @staticmethod + def _openai_system_message_to_anthropic( + message: dict[str, Any], + ) -> dict[str, Any] | None: # mutable-ok: API message payload + """Convert an OpenAI system message to the client's Anthropic-shaped entry.""" + content = message.get("content") + if isinstance(content, str): + return ( + {"role": "system", "content": content} if content else None # mutable-ok: API message payload + ) # mutable-ok: API message payload + if not isinstance(content, list): + return None + blocks: list[dict[str, Any]] = [] # mutable-ok: API message payload + for block in content: + if not isinstance(block, dict) or block.get("type") != "text": + continue + text = block.get("text") + if not isinstance(text, str) or not text: + continue + anthropic_block: dict[str, Any] = { # mutable-ok: API message payload + "type": "text", + "text": text, + } # mutable-ok: API message payload + cache_control = block.get("cache_control") + if cache_control: + anthropic_block["cache_control"] = deepcopy(cache_control) + blocks.append(anthropic_block) + return ( + {"role": "system", "content": blocks} if blocks else None # mutable-ok: API message payload + ) # mutable-ok: API message payload + + @staticmethod + def _is_hoisted_top_level_system(message: Any, hoisted_system_message: Any) -> bool: + """Match the hoisted prompt by identity, or by value after serialization.""" + if hoisted_system_message is None: + return False + if message is hoisted_system_message: + return True + return ( + isinstance(message, dict) and isinstance(hoisted_system_message, dict) and message == hoisted_system_message + ) + + @staticmethod + def _write_back_structured_messages( + data: dict, # mutable-ok: API message payload + structured_messages: list, # mutable-ok: API message payload + hoisted_system_message: Any = None, + preserve_system_messages: bool = False, + ) -> None: + """Write a guardrail's structured-message rewrite back without losing corrections.""" from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, group_tool_exchanges, ) + def _is_system(message: Any) -> bool: + return isinstance(message, dict) and str(message.get("role") or "").lower() == "system" + model = str(data.get("model") or "") - non_system = [m for m in structured_messages if m.get("role") != "system"] - groups = tuple([non_system[index] for index in group] for group in group_tool_exchanges(non_system)) or ( - non_system, - ) - converted = [ - message - for group in groups - for message in anthropic_messages_pt(messages=group, model=model, llm_provider="anthropic") - ] + converted: list = [] # mutable-ok: API message payload + + def _convert_run(run: list) -> None: # mutable-ok: API message payload + for group in group_tool_exchanges(run): + converted.extend( + anthropic_messages_pt( + messages=[ # mutable-ok: API message payload + run[index] for index in group + ], # mutable-ok: API message payload + model=model, + llm_provider="anthropic", + ) + ) + + run: list = [] # mutable-ok: API message payload + hoisted_dropped = False + for message in structured_messages: + if not _is_system(message): + run.append(message) + continue + _convert_run(run) + run = [] # mutable-ok: API message payload + if not hoisted_dropped and AnthropicMessagesHandler._is_hoisted_top_level_system( + message, hoisted_system_message + ): + hoisted_dropped = True + continue + if preserve_system_messages: + anthropic_system = AnthropicMessagesHandler._openai_system_message_to_anthropic(message) + if anthropic_system is not None: + converted.append(anthropic_system) + _convert_run(run) + if not any(not _is_system(message) for message in converted): + converted.extend( + anthropic_messages_pt( + messages=[], model=model, llm_provider="anthropic" + ) # mutable-ok: API message payload + ) # mutable-ok: API message payload for msg in converted: content = msg.get("content") if isinstance(content, list): @@ -397,6 +494,29 @@ class AnthropicMessagesHandler(BaseTranslation): block.pop("cache_control", None) data["messages"] = converted + @staticmethod + def _extract_midturn_system_text( + message: dict[str, Any], # mutable-ok: API message payload + msg_idx: int, + texts_to_check: list[str], # mutable-ok: API message payload + task_mappings: list[tuple[int, int | None]], # mutable-ok: API message payload + ) -> None: + content = message.get("content") + if isinstance(content, str): + if content: + texts_to_check.append(content) + task_mappings.append((msg_idx, None)) + return + if not isinstance(content, list): + return + for content_idx, content_item in enumerate(content): + if not isinstance(content_item, dict) or content_item.get("type") != "text": + continue + text_str = content_item.get("text") + if isinstance(text_str, str) and text_str: + texts_to_check.append(text_str) + task_mappings.append((msg_idx, content_idx)) + def extract_request_tool_names(self, data: dict) -> List[str]: """Extract tool names from Anthropic messages request (tools[].name).""" names: List[str] = [] @@ -415,15 +535,18 @@ class AnthropicMessagesHandler(BaseTranslation): skip_system_message: bool = False, skip_tool_message: bool = False, ) -> None: - """ - Extract text content and images from a message. - - Override this method to customize text/image extraction logic. - """ - role = str(message.get("role") or "").lower() - if skip_system_message and role == "system": + """Extract text content and images from a message.""" + role = str(message.get("role") or "") + if role == "system": + # Match the adapter's filtering so positional guardrail write-back stays aligned. + self._extract_midturn_system_text( + message=message, + msg_idx=msg_idx, + texts_to_check=texts_to_check, + task_mappings=task_mappings, + ) return - if skip_tool_message and role == "tool": + if skip_tool_message and role.lower() == "tool": return content = message.get("content", None) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 86c9c1db481..707d53e9006 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -85,12 +85,12 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import ) from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, + AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, - AnthopicMessagesAssistantMessageParam, AnthropicFinishReason, AnthropicMessagesRequest, + AnthropicMessagesSystemMessageParam, AnthropicMessagesToolChoice, - AnthropicMessagesUserMessageParam, AnthropicResponseContentBlockRedactedThinking, AnthropicResponseContentBlockText, AnthropicResponseContentBlockThinking, @@ -354,12 +354,7 @@ class LiteLLMAnthropicMessagesAdapter: def translate_anthropic_messages_to_openai( self, - messages: List[ - Union[ - AnthropicMessagesUserMessageParam, - AnthopicMessagesAssistantMessageParam, - ] - ], + messages: List[AllAnthropicPassThroughMessageValues], # mutable-ok: API message payload model: Optional[str] = None, ) -> List: new_messages: List[AllMessageValues] = [] @@ -367,6 +362,11 @@ class LiteLLMAnthropicMessagesAdapter: user_message: Optional[ChatCompletionUserMessage] = None tool_message_list: List[ChatCompletionToolMessage] = [] new_user_content_list: List[Union[ChatCompletionTextObject, ChatCompletionImageObject]] = [] + if m["role"] == "system": + system_message = self._translate_midturn_system_message_to_openai(m, model) + if system_message is not None: + new_messages.append(system_message) + continue ## USER MESSAGE ## if m["role"] == "user": ## translate user message @@ -867,6 +867,29 @@ class LiteLLMAnthropicMessagesAdapter: for def_schema in schema[key].values(): LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) + def _translate_midturn_system_message_to_openai( + self, + message: AnthropicMessagesSystemMessageParam, + model: str | None, + ) -> ChatCompletionSystemMessage | None: + """Translate an in-sequence system entry without changing its role or position.""" + content = message.get("content") + if isinstance(content, str): + return ChatCompletionSystemMessage(role="system", content=content) if content else None + if not isinstance(content, list): + return None + text_parts: list[ChatCompletionTextObject] = [] # mutable-ok: API message payload + for block in content: + if not isinstance(block, dict) or block.get("type") != "text": + continue + text = block.get("text") + if not text: + continue + text_obj = ChatCompletionTextObject(type="text", text=text) + self._add_cache_control_if_applicable(block, text_obj, model) + text_parts.append(text_obj) + return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None + def _add_system_message_to_messages( self, new_messages: List[AllMessageValues], @@ -1068,13 +1091,8 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping: Dict[str, str] = {} ## CONVERT ANTHROPIC MESSAGES TO OPENAI - messages_list: List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]] = cast( - List[ - Union[ - AnthropicMessagesUserMessageParam, - AnthopicMessagesAssistantMessageParam, - ] - ], + messages_list = cast( + List[AllAnthropicPassThroughMessageValues], anthropic_message_request["messages"], ) new_messages = self.translate_anthropic_messages_to_openai( diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 172e54de98e..cbe36100eb0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,6 +6,7 @@ path used for OpenAI and Azure models. """ import json +from collections.abc import Iterable from typing import Any, Dict, List, Optional, Union, cast from litellm.litellm_core_utils.reasoning_effort_utils import ( @@ -15,15 +16,15 @@ from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, ) from litellm.types.llms.anthropic import ( + AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, - AnthopicMessagesAssistantMessageParam, AnthropicFinishReason, AnthropicMessagesRequest, AnthropicMessagesToolChoice, - AnthropicMessagesUserMessageParam, AnthropicResponseContentBlockText, AnthropicResponseContentBlockThinking, AnthropicResponseContentBlockToolUse, + AnthropicSystemMessageContent, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -54,19 +55,32 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return source.get("url") return None + @staticmethod + def _translate_midturn_system_content_to_responses( + content: Union[str, Iterable[AnthropicSystemMessageContent]], + ) -> list[dict[str, str]]: # mutable-ok: API message payload + """Convert in-sequence system content to Responses input-text parts.""" + if isinstance(content, str): + return ( + [{"type": "input_text", "text": content}] if content else [] # mutable-ok: API message payload + ) # mutable-ok: API message payload + if not isinstance(content, list): + return [] # mutable-ok: API message payload + return [ # mutable-ok: API message payload + {"type": "input_text", "text": text} # mutable-ok: API message payload + for block in content + if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) + ] + def translate_messages_to_responses_input( self, - messages: List[ - Union[ - AnthropicMessagesUserMessageParam, - AnthopicMessagesAssistantMessageParam, - ] - ], + messages: List[AllAnthropicPassThroughMessageValues], # mutable-ok: API message payload ) -> List[Dict[str, Any]]: """ Convert Anthropic messages list to Responses API `input` items. Mapping: + system text -> message(role=system, input_text) user text -> message(role=user, input_text) user image -> message(role=user, input_image) user tool_result -> function_call_output @@ -76,6 +90,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: input_items: List[Dict[str, Any]] = [] for m in messages: + if m["role"] == "system": + system_parts = self._translate_midturn_system_content_to_responses(m.get("content")) + if system_parts: + input_items.append( + { # mutable-ok: API message payload + "type": "message", + "role": "system", + "content": system_parts, + } + ) + continue + role = m["role"] content = m.get("content") @@ -287,12 +313,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: """ model: str = anthropic_request["model"] messages_list = cast( - List[ - Union[ - AnthropicMessagesUserMessageParam, - AnthopicMessagesAssistantMessageParam, - ] - ], + List[AllAnthropicPassThroughMessageValues], anthropic_request["messages"], ) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index af419d8cb6f..2e0da24ccda 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -11,12 +11,24 @@ from litellm.types.proxy.guardrails.guardrail_hooks.akto import ( from litellm.types.proxy.guardrails.guardrail_hooks.block_code_execution import ( BlockCodeExecutionGuardrailConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.cisco_ai_defense import ( + CiscoAIDefenseGuardrailConfigModel, +) +from litellm.types.proxy.guardrails.guardrail_hooks.compresr import ( + CompresrGuardrailConfigModel, +) from litellm.types.proxy.guardrails.guardrail_hooks.enkryptai import ( EnkryptAIGuardrailConfigs, ) from litellm.types.proxy.guardrails.guardrail_hooks.grayswan import ( GraySwanGuardrailConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.headroom import ( + HeadroomGuardrailConfigModel, +) +from litellm.types.proxy.guardrails.guardrail_hooks.hiddenlayer import ( + HiddenlayerGuardrailConfigModel, +) from litellm.types.proxy.guardrails.guardrail_hooks.ibm import ( IBMGuardrailsBaseConfigModel, ) @@ -29,38 +41,26 @@ from litellm.types.proxy.guardrails.guardrail_hooks.ovalix import ( from litellm.types.proxy.guardrails.guardrail_hooks.promptguard import ( PromptGuardConfigModel, ) -from litellm.types.proxy.guardrails.guardrail_hooks.xecguard import ( - XecGuardConfigModel, +from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, ) from litellm.types.proxy.guardrails.guardrail_hooks.qualifire import ( QualifireGuardrailConfigModel, ) -from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( - ToolPermissionGuardrailConfigModel, -) -from litellm.types.proxy.guardrails.guardrail_hooks.hiddenlayer import ( - HiddenlayerGuardrailConfigModel, -) -from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( - QostodianNexusConfigModel, -) from litellm.types.proxy.guardrails.guardrail_hooks.repelloai import ( RepelloAIGuardrailConfigModel, ) -from litellm.types.proxy.guardrails.guardrail_hooks.vigil_guard import ( - VigilGuardGuardrailConfigModel, -) -from litellm.types.proxy.guardrails.guardrail_hooks.cisco_ai_defense import ( - CiscoAIDefenseGuardrailConfigModel, -) from litellm.types.proxy.guardrails.guardrail_hooks.singulr import ( SingulrGuardrailConfigModel, ) -from litellm.types.proxy.guardrails.guardrail_hooks.headroom import ( - HeadroomGuardrailConfigModel, +from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( + ToolPermissionGuardrailConfigModel, ) -from litellm.types.proxy.guardrails.guardrail_hooks.compresr import ( - CompresrGuardrailConfigModel, +from litellm.types.proxy.guardrails.guardrail_hooks.vigil_guard import ( + VigilGuardGuardrailConfigModel, +) +from litellm.types.proxy.guardrails.guardrail_hooks.xecguard import ( + XecGuardConfigModel, ) """ @@ -743,7 +743,10 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up "When True, unified guardrails skip system-role messages when building " "evaluation inputs (texts and structured_messages). When False, system " "messages are included even if litellm_settings sets a global skip. When " - "None, use the global litellm.skip_system_message_in_guardrail setting." + "None, use the global litellm.skip_system_message_in_guardrail setting. " + "For Anthropic /v1/messages, the flag applies only to the trusted top-level " + "system prompt. In-sequence system entries are untrusted client input and remain " + "in texts and structured_messages." ), ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index c24d072217a..29faf500b3d 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -365,8 +365,20 @@ class AnthropicSystemMessageContent(TypedDict, total=False): cache_control: Optional[Union[dict, ChatCompletionCachedContent]] +class AnthropicMessagesSystemMessageParam(TypedDict, total=False): + role: Required[Literal["system"]] + content: Required[Union[str, Iterable[AnthropicSystemMessageContent]]] + + AllAnthropicMessageValues = Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam] +# System is not a native Anthropic message role; only pass-through adapters use this union. +AllAnthropicPassThroughMessageValues = Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + AnthropicMessagesSystemMessageParam, +] + class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): max_tokens: Optional[int] diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 48acdd348e9..7757b0fa5a4 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -75,6 +75,51 @@ class MockRecordingGuardrail(CustomGuardrail): return inputs +class MockMaskingGuardrail(CustomGuardrail): + """Capture request inputs and mask one known prohibited value.""" + + def __init__(self, skip_system_message_in_guardrail: Optional[bool] = True): + super().__init__(guardrail_name="masking-test") + self.skip_system_message_in_guardrail = skip_system_message_in_guardrail + self.inputs: Optional[GenericGuardrailAPIInputs] = None + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + self.inputs = inputs.copy() + masked_inputs = inputs.copy() + masked_inputs["texts"] = [ + "[MASKED]" if text == "prohibited correction" else text for text in inputs.get("texts", []) + ] + return masked_inputs + + +class MockCompactingGuardrail(CustomGuardrail): + """Stand in for a compaction guardrail that rewrites `structured_messages` wholesale.""" + + def __init__(self, replacement_messages: list): + super().__init__(guardrail_name="compacting-test") + self.replacement_messages = replacement_messages + self.inputs: Optional[GenericGuardrailAPIInputs] = None + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + self.inputs = inputs.copy() + rewritten = inputs.copy() + # A new list object -- this is what signals a rewrite to the handler. + rewritten["structured_messages"] = list(self.replacement_messages) + return rewritten + + class TestAnthropicMessagesHandlerStreamingRequestData: """Post-call guardrails on streaming /v1/messages receive the response and identity metadata""" @@ -210,6 +255,660 @@ class TestAnthropicMessagesHandlerInputProcessing: assert data.get("litellm_metadata", {}).get("guardrails") assert guardrail.dynamic_params == {"policy_id": "policy-123"} + @pytest.mark.asyncio + async def test_midturn_system_correction_is_guardrailed_when_top_level_system_is_skipped( + self, + ): + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "safe text"}, + { + "role": "system", + "content": [ + {"type": "unsupported", "text": "discarded text"}, + {"type": "text", "text": "prohibited correction"}, + ], + }, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + assert guardrail.inputs["texts"] == ["safe text", "prohibited correction"] + assert "trusted top-level system prompt" not in guardrail.inputs["texts"] + assert data["messages"][1]["content"][0]["text"] == "discarded text" + assert data["messages"][1]["content"][1]["text"] == "[MASKED]" + + @pytest.mark.asyncio + async def test_string_midturn_system_correction_is_guardrailed(self): + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [{"role": "system", "content": "prohibited correction"}], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + assert guardrail.inputs["texts"] == ["prohibited correction"] + assert data["messages"][0]["content"] == "[MASKED]" + + @pytest.mark.asyncio + async def test_unsupported_midturn_system_content_is_not_guardrailed(self): + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + { + "role": "system", + "content": [{"type": "image", "source": {"type": "url"}}], + } + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is None + + @pytest.mark.asyncio + async def test_skip_system_message_excludes_only_hoisted_top_level_system(self): + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "safe text"}, + {"role": "system", "content": "prohibited correction"}, + {"role": "user", "content": "continue"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + structured = guardrail.inputs["structured_messages"] + assert [m["role"] for m in structured] == ["user", "system", "user"] + assert structured[1]["content"] == "prohibited correction" + + @pytest.mark.asyncio + async def test_default_skip_false_scans_midturn_system_and_hoists_top_level_system( + self, + ): + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail(skip_system_message_in_guardrail=None) + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "safe text"}, + {"role": "system", "content": "prohibited correction"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + assert guardrail.inputs["texts"] == ["safe text", "prohibited correction"] + structured = guardrail.inputs["structured_messages"] + assert [m["role"] for m in structured] == ["system", "user", "system"] + assert structured[0]["content"] == "trusted top-level system prompt" + assert data["messages"][1]["content"] == "[MASKED]" + + @pytest.mark.asyncio + async def test_bedrock_masking_slice_is_unavailable_when_top_level_system_is_included( + self, + ): + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail(skip_system_message_in_guardrail=None) + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "safe text"}, + {"role": "system", "content": "prohibited correction"}, + {"role": "user", "content": "latest question"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + texts = guardrail.inputs["texts"] + structured = guardrail.inputs["structured_messages"] + + bedrock = BedrockGuardrail(guardrailIdentifier="gi", guardrailVersion="1") + assert sum(bedrock._count_message_texts(m) for m in structured) == len(texts) + 1 + latest_user_index = bedrock._find_latest_message_index(structured, target_role="user") + assert ( + bedrock._locate_message_texts_slice( + structured_messages=structured, + target_index=latest_user_index, + texts=texts, + ) + is None + ) + assert ( + bedrock._merge_masked_texts( + masked_texts=["{MASKED}"], + texts=texts, + scanned_slice=None, + scanned_role_subset=True, + ) + == texts + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("skip_system_message_in_guardrail", [True, None]) + async def test_midturn_system_text_extraction_matches_translation_in_both_skip_modes( + self, + skip_system_message_in_guardrail: Optional[bool], + ): + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail(skip_system_message_in_guardrail=skip_system_message_in_guardrail) + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "safe text"}, + { + "role": "system", + "content": [ + {"type": "text", "text": ""}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}, + {"type": "text", "text": "prohibited correction"}, + ], + }, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + texts = guardrail.inputs["texts"] + structured = guardrail.inputs["structured_messages"] + assert texts == ["safe text", "prohibited correction"] + bedrock = BedrockGuardrail(guardrailIdentifier="gi", guardrailVersion="1") + assert sum(bedrock._count_message_texts(m) for m in structured) == len(texts) + assert data["messages"][1]["content"][2]["text"] == "[MASKED]" + + @pytest.mark.asyncio + async def test_bedrock_masking_slice_stays_aligned_with_midturn_system(self): + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "safe text"}, + { + "role": "system", + "content": [ + {"type": "text", "text": "prohibited correction"}, + {"type": "text", "text": "second correction"}, + ], + }, + {"role": "user", "content": "latest question"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + texts = guardrail.inputs["texts"] + structured = guardrail.inputs["structured_messages"] + + bedrock = BedrockGuardrail(guardrailIdentifier="gi", guardrailVersion="1") + total = sum(bedrock._count_message_texts(m) for m in structured) + assert total == len(texts) + + latest_user_index = bedrock._find_latest_message_index(structured, target_role="user") + assert latest_user_index == 2 + scanned_slice = bedrock._locate_message_texts_slice( + structured_messages=structured, + target_index=latest_user_index, + texts=texts, + ) + assert scanned_slice == (3, 1) + + merged = bedrock._merge_masked_texts( + masked_texts=["{MASKED}"], + texts=texts, + scanned_slice=scanned_slice, + scanned_role_subset=True, + ) + assert merged == [ + "safe text", + "prohibited correction", + "second correction", + "{MASKED}", + ] + + @pytest.mark.asyncio + async def test_compaction_rewrite_keeps_midturn_system_messages(self): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "user", "content": "compacted history"}, + { + "role": "system", + "content": [{"type": "text", "text": "use the corrected result"}], + }, + {"role": "user", "content": "continue"}, + ] + ) + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "continue"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["user", "system", "user"] + assert data["messages"][1]["content"] == [{"type": "text", "text": "use the corrected result"}] + assert data["messages"][0]["content"] == [{"type": "text", "text": "compacted history"}] + assert data["messages"][2]["content"] == [{"type": "text", "text": "continue"}] + assert data["system"] == "trusted top-level system prompt" + + @pytest.mark.asyncio + async def test_compaction_rewrite_does_not_duplicate_hoisted_top_level_system(self): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "system", "content": "trusted top-level system prompt"}, + {"role": "user", "content": "compacted history"}, + {"role": "system", "content": "use the corrected result"}, + ] + ) + guardrail.skip_system_message_in_guardrail = None + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["user", "system"] + assert data["messages"][1]["content"] == "use the corrected result" + assert data["system"] == "trusted top-level system prompt" + + @pytest.mark.asyncio + async def test_compaction_rewrite_keeps_leading_midturn_system_when_system_is_skipped( + self, + ): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "compacted history"}, + ] + ) + guardrail.skip_system_message_in_guardrail = True + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "trusted top-level system prompt", + "messages": [ + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "original history"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["system", "user"] + assert data["messages"][0]["content"] == "use the corrected result" + + @pytest.mark.asyncio + async def test_compaction_rewrite_keeps_leading_correction_when_top_level_system_hoists_nothing( + self, + ): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "compacted history"}, + ] + ) + guardrail.skip_system_message_in_guardrail = None + data = { + "model": "claude-3-5-sonnet-20241022", + "system": [{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}], + "messages": [ + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "original history"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["system", "user"] + assert data["messages"][0]["content"] == "use the corrected result" + + @pytest.mark.asyncio + async def test_compaction_rewrite_keeps_leading_correction_when_hoisted_prompt_is_dropped( + self, + ): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "system", "content": "CLIENT CORRECTION"}, + {"role": "user", "content": "compacted history"}, + ] + ) + guardrail.skip_system_message_in_guardrail = None + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "TRUSTED", + "messages": [ + {"role": "system", "content": "CLIENT CORRECTION"}, + {"role": "user", "content": "original history"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + assert guardrail.inputs["structured_messages"][0] == { + "role": "system", + "content": "TRUSTED", + } + assert [m["role"] for m in data["messages"]] == ["system", "user"] + assert data["messages"][0]["content"] == "CLIENT CORRECTION" + assert data["system"] == "TRUSTED" + + @pytest.mark.asyncio + async def test_compaction_rewrite_drops_hoisted_prompt_matched_by_content_copy(self): + import json + + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + json.loads(json.dumps({"role": "system", "content": "TRUSTED"})), + {"role": "user", "content": "compacted history"}, + {"role": "system", "content": "CLIENT CORRECTION"}, + ] + ) + guardrail.skip_system_message_in_guardrail = None + data = { + "model": "claude-3-5-sonnet-20241022", + "system": "TRUSTED", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "CLIENT CORRECTION"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["user", "system"] + assert data["messages"][1]["content"] == "CLIENT CORRECTION" + assert data["system"] == "TRUSTED" + + @pytest.mark.asyncio + async def test_compaction_rewrite_preserves_cache_control_on_system_blocks(self): + """ + `cache_control` on an in-sequence system text block survives the write-back, and is + copied rather than aliased into the guardrail's own returned list. + """ + handler = AnthropicMessagesHandler() + source_cache_control = {"type": "ephemeral"} + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "user", "content": "compacted history"}, + { + "role": "system", + "content": [ + { + "type": "text", + "text": "use the corrected result", + "cache_control": source_cache_control, + } + ], + }, + ] + ) + guardrail.skip_system_message_in_guardrail = True + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert data["messages"][1]["content"] == [ + { + "type": "text", + "text": "use the corrected result", + "cache_control": {"type": "ephemeral"}, + } + ] + assert data["messages"][1]["content"][0]["cache_control"] is not source_cache_control + + @pytest.mark.asyncio + async def test_compaction_rewrite_rstrips_trailing_assistant_in_each_run(self): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "user", "content": "compacted history"}, + {"role": "assistant", "content": "earlier "}, + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "continue"}, + {"role": "assistant", "content": "prefill "}, + ] + ) + guardrail.skip_system_message_in_guardrail = True + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == [ + "user", + "assistant", + "system", + "user", + "assistant", + ] + assert data["messages"][1]["content"] == [{"type": "text", "text": "earlier"}] + assert data["messages"][-1]["content"] == [{"type": "text", "text": "prefill"}] + + @pytest.mark.asyncio + async def test_compaction_rewrite_drops_text_free_system_message(self): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[ + {"role": "user", "content": "compacted history"}, + {"role": "system", "content": [{"type": "text", "text": ""}]}, + {"role": "system", "content": ""}, + {"role": "user", "content": "continue"}, + ] + ) + guardrail.skip_system_message_in_guardrail = True + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "continue"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["user", "user"] + assert data["messages"][0]["content"] == [{"type": "text", "text": "compacted history"}] + assert data["messages"][1]["content"] == [{"type": "text", "text": "continue"}] + + @pytest.mark.asyncio + async def test_noncanonical_system_role_casing_is_still_scanned(self): + handler = AnthropicMessagesHandler() + guardrail = MockMaskingGuardrail() + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "safe text"}, + {"role": "System", "content": "prohibited correction"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.inputs is not None + assert "prohibited correction" in guardrail.inputs["texts"] + assert data["messages"][1]["content"] == "[MASKED]" + + @pytest.mark.asyncio + async def test_tool_result_turns_have_a_preexisting_alignment_gap(self): + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + + handler = AnthropicMessagesHandler() + bedrock = BedrockGuardrail(guardrailIdentifier="gi", guardrailVersion="1") + tool_loop = [ + {"role": "user", "content": "call the tool"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu_1", "name": "get", "input": {"a": 1}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "text", "text": "tool output"}], + } + ], + }, + ] + + async def _slice_for(messages: list): + guardrail = MockMaskingGuardrail() + data = {"model": "claude-3-5-sonnet-20241022", "messages": messages} + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + assert guardrail.inputs is not None + texts = guardrail.inputs["texts"] + structured = guardrail.inputs["structured_messages"] + target_index = bedrock._find_latest_message_index(structured, target_role="user") + return ( + sum(bedrock._count_message_texts(m) for m in structured) - len(texts), + bedrock._locate_message_texts_slice( + structured_messages=structured, + target_index=target_index, + texts=texts, + ), + ) + + with_system = await _slice_for( + tool_loop + + [ + {"role": "system", "content": "use the corrected result"}, + {"role": "user", "content": "latest question"}, + ] + ) + without_system = await _slice_for(tool_loop + [{"role": "user", "content": "latest question"}]) + + assert with_system == without_system == (1, None) + + @pytest.mark.asyncio + async def test_compaction_rewrite_to_only_system_messages_is_rejected(self): + import litellm + + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[{"role": "system", "content": "use the corrected result"}] + ) + guardrail.skip_system_message_in_guardrail = True + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + ], + } + + with patch.object(litellm, "modify_params", False): + with pytest.raises(litellm.BadRequestError, match="at least one non-system message"): + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + @pytest.mark.asyncio + async def test_compaction_rewrite_to_only_system_messages_repaired_with_modify_params( + self, + ): + import litellm + + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail( + replacement_messages=[{"role": "system", "content": "use the corrected result"}] + ) + guardrail.skip_system_message_in_guardrail = True + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "original history"}, + {"role": "system", "content": "use the corrected result"}, + ], + } + + with patch.object(litellm, "modify_params", True): + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["system", "user"] + assert data["messages"][0]["content"] == "use the corrected result" + + @pytest.mark.asyncio + async def test_compaction_rewrite_without_system_messages_is_unchanged(self): + handler = AnthropicMessagesHandler() + guardrail = MockCompactingGuardrail(replacement_messages=[{"role": "user", "content": "compacted history"}]) + data = { + "model": "claude-3-5-sonnet-20241022", + "messages": [ + {"role": "user", "content": "a"}, + {"role": "assistant", "content": "b"}, + {"role": "user", "content": "c"}, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert data["messages"] == [{"role": "user", "content": [{"type": "text", "text": "compacted history"}]}] + @pytest.mark.asyncio async def test_process_output_streaming_response_empty_choices(self): """Test that streaming response with empty choices doesn't raise IndexError diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index c0c6e315b5b..b72620f9918 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -413,6 +413,224 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement(): ), "Tool message should be placed before user message" +@pytest.mark.parametrize( + ("system_content", "expected_content"), + [ + ("Use the corrected result.", "Use the corrected result."), + ( + [{"type": "text", "text": "Use the corrected result."}], + [{"type": "text", "text": "Use the corrected result."}], + ), + ( + [ + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/a.png"}, + }, + {"type": "text", "text": "Use the corrected result."}, + ], + [{"type": "text", "text": "Use the corrected result."}], + ), + ( + [ + {"type": "text", "text": "First correction."}, + {"type": "text", "text": "Second correction."}, + ], + [ + {"type": "text", "text": "First correction."}, + {"type": "text", "text": "Second correction."}, + ], + ), + ], +) +def test_translate_anthropic_messages_to_openai_preserves_midturn_system_correction( + system_content: object, + expected_content: object, +): + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01234", + "name": "get_weather", + "input": {"location": "Boston"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01234", + "content": "Rainy, 55°F", + } + ], + }, + {"role": "system", "content": system_content}, + {"role": "user", "content": "Continue."}, + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="claude-3-5-sonnet-20240620", + ) + + assert result == [ + { + "role": "assistant", + "content": None, + "thinking_blocks": None, + "tool_calls": [ + { + "id": "toolu_01234", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Boston"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "toolu_01234", + "content": "Rainy, 55°F", + }, + {"role": "system", "content": expected_content}, + {"role": "user", "content": "Continue."}, + ] + + +def test_translate_anthropic_messages_to_openai_preserves_midturn_system_cache_control(): + """ + `cache_control` on an in-sequence system text block survives, matching how the + hoisted top-level `system` prompt and user text blocks are already handled. + """ + messages = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Use the corrected result.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="claude-3-5-sonnet-20240620", + ) + + assert result == [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Use the corrected result.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + +def test_translate_anthropic_messages_to_openai_drops_midturn_system_cache_control_for_non_claude(): + """ + `cache_control` goes through the same `_add_cache_control_if_applicable` gate as the + hoisted top-level prompt and user text blocks, so a non-Claude *requested model name* + does not get it. That gate is a best-effort check of the requested name before routing + (behind the proxy it is often a public alias), not a guarantee about the backend that + ultimately serves the request. + """ + messages = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Use the corrected result.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="gpt-4o", + ) + + assert result == [ + { + "role": "system", + "content": [{"type": "text", "text": "Use the corrected result."}], + } + ] + + +@pytest.mark.parametrize( + "system_content", + [ + "", + [{"type": "text", "text": ""}], + [ + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/a.png"}, + } + ], + None, + ], +) +def test_translate_anthropic_messages_to_openai_drops_empty_midturn_system( + system_content: object, +): + messages = [{"role": "system", "content": system_content}] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="claude-3-5-sonnet-20240620", + ) + + assert result == [] + + +def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system(): + """ + Request level: the trusted top-level prompt is hoisted to index 0 exactly once and the + in-sequence correction keeps its own position and `role: "system"` -- no duplication of + either, and no reordering of the surrounding turns. + """ + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={ + "model": "claude-3-5-sonnet-20240620", + "max_tokens": 100, + "system": "Trusted top-level prompt.", + "messages": [ + {"role": "user", "content": "First question."}, + {"role": "assistant", "content": "First answer."}, + {"role": "system", "content": "Use the corrected result."}, + {"role": "user", "content": "Continue."}, + ], + } + ) + + assert openai_request["messages"] == [ + {"role": "system", "content": "Trusted top-level prompt."}, + {"role": "user", "content": "First question."}, + {"role": "assistant", "content": "First answer.", "thinking_blocks": None}, + {"role": "system", "content": "Use the corrected result."}, + {"role": "user", "content": "Continue."}, + ] + + def test_translate_openai_content_to_anthropic_empty_function_arguments(): """Test that empty function arguments are handled safely and don't cause JSON parsing errors.""" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py index 9c8df1c79f9..6cc1d9e5add 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py @@ -2475,3 +2475,57 @@ def test_endpoint_runs_failure_hook_on_500_context_management_error(): body = response.json() assert body["type"] == "error" failure_hook.assert_awaited_once() + + +def test_count_effective_tokens_counts_midturn_system_correction(): + from litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact import ( + _count_effective_tokens, + ) + + base: List[Dict[str, Any]] = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ] + correction = { + "role": "system", + "content": [{"type": "text", "text": "use the corrected result " * 20}], + } + + without_correction = _count_effective_tokens( + model=MODEL, effective_messages=base, compaction_block=None, tools=None + ) + with_correction = _count_effective_tokens( + model=MODEL, + effective_messages=base + [correction], + compaction_block=None, + tools=None, + ) + + assert with_correction > without_correction + + +def test_build_summary_messages_keeps_midturn_system_correction_in_place(): + from litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact import ( + _build_summary_messages, + ) + + summary_messages = _build_summary_messages( + effective_messages=[ + {"role": "user", "content": "original question"}, + {"role": "system", "content": "use the corrected result"}, + {"role": "assistant", "content": "acknowledged"}, + ], + prompt="summarize the conversation", + system="caller system prompt", + ) + + assert [m["role"] for m in summary_messages] == [ + "system", + "user", + "system", + "assistant", + "user", + ] + assert summary_messages[0]["content"] == "caller system prompt" + assert summary_messages[2]["content"] == "use the corrected result" + assert summary_messages[-1]["content"] == "summarize the conversation" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 606ff39b35e..8963012ecd5 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -9,6 +9,8 @@ import sys from typing import Any, Dict, List from unittest.mock import MagicMock +import pytest + sys.path.insert(0, os.path.abspath("../../../../../../..")) from litellm.constants import ( @@ -221,6 +223,106 @@ class TestTranslateMessagesToResponsesInput: {"type": "input_text", "text": "Second part."}, ] + @pytest.mark.parametrize( + "system_content", + [ + "Use the corrected result.", + [{"type": "text", "text": "Use the corrected result."}], + [ + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}, + {"type": "text", "text": "Use the corrected result."}, + ], + ], + ) + def test_midturn_system_correction_stays_system_in_sequence(self, system_content: object): + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01234", + "name": "get_weather", + "input": {"location": "Boston"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01234", + "content": "Rainy, 55°F", + } + ], + }, + {"role": "system", "content": system_content}, + {"role": "user", "content": "Continue."}, + ] + + result = _translate_messages(messages) + + assert result == [ + { + "type": "function_call", + "call_id": "toolu_01234", + "name": "get_weather", + "arguments": '{"location": "Boston"}', + }, + { + "type": "function_call_output", + "call_id": "toolu_01234", + "output": "Rainy, 55°F", + }, + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "Use the corrected result."}], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Continue."}], + }, + ] + + def test_midturn_system_correction_keeps_multiple_text_blocks(self): + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "First correction."}, + {"type": "text", "text": "Second correction."}, + ], + } + ] + + assert _translate_messages(messages) == [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "input_text", "text": "First correction."}, + {"type": "input_text", "text": "Second correction."}, + ], + } + ] + + @pytest.mark.parametrize( + "system_content", + [ + "", + [{"type": "text", "text": ""}], + [{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}], + None, + ], + ) + def test_empty_or_unsupported_midturn_system_correction_is_dropped(self, system_content: object): + messages = [{"role": "system", "content": system_content}] + + assert _translate_messages(messages) == [] + def test_user_base64_image(self): """User message with base64 image source becomes input_image with data URL.""" messages = [ @@ -722,6 +824,42 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) assert kwargs["instructions"] == "You are a helpful assistant." + def test_top_level_system_and_midturn_correction_are_not_duplicated(self): + """ + Request level: the trusted top-level prompt goes to `instructions` only, and the + in-sequence correction stays a `role: "system"` input item in its original position. + Neither appears twice, and the surrounding turns keep their order. + """ + req = _make_request( + system="Trusted top-level prompt.", + messages=[ + {"role": "user", "content": "First question."}, + {"role": "system", "content": "Use the corrected result."}, + {"role": "user", "content": "Continue."}, + ], + ) + + kwargs = _ADAPTER.translate_request(req) + + assert kwargs["instructions"] == "Trusted top-level prompt." + assert kwargs["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "First question."}], + }, + { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": "Use the corrected result."}], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Continue."}], + }, + ] + def test_system_list_of_text_blocks_joined(self): req = _make_request( system=[ From c5c5a276790529e2de3378654864fd847530c5a0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:19:42 +0000 Subject: [PATCH 029/672] fix(files): enforce require_managed_files on file retrieve, content and delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../openai_files_endpoints/common_utils.py | 30 +++++ .../openai_files_endpoints/files_endpoints.py | 7 ++ .../test_files_endpoint.py | 116 ++++++++++++++++++ 3 files changed, 153 insertions(+) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 87514b46dbd..3eef3868c94 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -866,6 +866,36 @@ def validate_managed_files_requirement( ) +def validate_managed_file_id_requirement(file_id: str) -> None: + """ + Enforce proxy-level managed files on the file read/delete routes when + ``litellm.require_managed_files`` is enabled. + + Ownership is only recorded for LiteLLM managed files, so a raw provider file id sent to + retrieve/content/delete is forwarded to the provider under shared credentials without any + tenant check; knowing another tenant's provider file id would be enough to read or delete it. + + Raises: + HTTPException: 400 if ``file_id`` is not a LiteLLM managed file id. + """ + import litellm + from fastapi import HTTPException + + if litellm.require_managed_files is not True: + return + + if _is_base64_encoded_unified_file_id(file_id): + return + + raise HTTPException( + status_code=400, + detail=( + "Raw provider file ids cannot be used when require_managed_files is enabled in " + "litellm_settings. Use the LiteLLM managed file id returned when the file was created." + ), + ) + + def _extract_model_param(request: "Request", request_body: dict) -> str | None: """ Extract model parameter from request. diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 4e4718272bd..37f1ced6996 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( get_credentials_for_model, handle_model_based_routing, prepare_data_with_credentials, + validate_managed_file_id_requirement, validate_managed_files_requirement, ) from litellm.proxy.utils import ProxyLogging, is_known_model @@ -612,6 +613,8 @@ async def get_file_content( data: dict = {"file_id": file_id} try: + validate_managed_file_id_requirement(file_id=file_id) + # Include original request and headers in the data base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) ( @@ -908,6 +911,8 @@ async def get_file( data: dict = {"file_id": file_id} try: + validate_managed_file_id_requirement(file_id=file_id) + custom_llm_provider = ( provider or get_custom_llm_provider_from_request_headers(request=request) @@ -1098,6 +1103,8 @@ async def delete_file( data: dict = {"file_id": file_id} try: + validate_managed_file_id_requirement(file_id=file_id) + custom_llm_provider = ( provider or get_custom_llm_provider_from_request_headers(request=request) diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index ac01c6ae1d1..24b814bae1f 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -3051,3 +3051,119 @@ def test_list_files_key_allowed_openai_model_still_resolves_team_credentials( mocker, monkeypatch, _team_openai_plus_global_anthropic_router(), ["team-gpt"] ) assert captured_kwargs.get("api_key") == "team-openai-key" + + +@pytest.mark.parametrize( + "http_method, url, patched_litellm_call", + [ + ("get", "/v1/files/file-victim-abc123", "litellm.afile_retrieve"), + ("get", "/v1/files/file-victim-abc123/content", "litellm.afile_content"), + ("delete", "/v1/files/file-victim-abc123", "litellm.afile_delete"), + ], +) +def test_require_managed_files_rejects_raw_provider_file_id( + mocker: MockerFixture, + monkeypatch, + llm_router: Router, + http_method: str, + url: str, + patched_litellm_call: str, +): + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setattr("litellm.require_managed_files", True) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router) + setup_proxy_logging_object(monkeypatch, llm_router) + + mock_call = mocker.patch(patched_litellm_call, new=mocker.AsyncMock()) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="attacker-user" + ) + + try: + response = getattr(client, http_method)( + url, headers={"Authorization": "Bearer test-key"} + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + monkeypatch.setattr("litellm.require_managed_files", False) + + assert response.status_code == 400, response.text + mock_call.assert_not_called() + + +def _unified_managed_file_id() -> str: + import base64 + + from litellm.types.utils import SpecialEnums + + unified_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( + "application/json", "victim-unified-id", "gpt-3.5-turbo", "file-victim-abc123", "gpt-3.5-turbo-id" + ) + return base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=") + + +def test_require_managed_files_allows_unified_managed_file_id(monkeypatch): + from litellm.proxy.openai_files_endpoints.common_utils import ( + validate_managed_file_id_requirement, + ) + + monkeypatch.setattr("litellm.require_managed_files", True) + + validate_managed_file_id_requirement(file_id=_unified_managed_file_id()) + + +def test_managed_file_id_requirement_is_opt_in(monkeypatch): + from litellm.proxy.openai_files_endpoints.common_utils import ( + validate_managed_file_id_requirement, + ) + + monkeypatch.setattr("litellm.require_managed_files", False) + + validate_managed_file_id_requirement(file_id="file-victim-abc123") + + +def test_raw_provider_file_id_retrieve_allowed_when_managed_files_not_required( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setattr("litellm.require_managed_files", False) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router) + setup_proxy_logging_object(monkeypatch, llm_router) + + mock_retrieve = mocker.patch( + "litellm.afile_retrieve", + new=mocker.AsyncMock( + return_value=OpenAIFileObject( + id="file-victim-abc123", + object="file", + bytes=3, + created_at=1234567890, + filename="test.txt", + purpose="user_data", + status="uploaded", + ) + ), + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="some-user" + ) + + try: + response = client.get( + "/v1/files/file-victim-abc123", headers={"Authorization": "Bearer test-key"} + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + mock_retrieve.assert_called_once() From 81a80b8c632a9b5d942193bb71eda601630e9906 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:02:27 +0000 Subject: [PATCH 030/672] fix(anthropic): preserve speed=fast in usage for /v1/messages and pass-through Fast mode is priced with a provider-specific multiplier applied off usage.speed, but only chat completions kept that field. The Messages route rebuilt usage with empty optional params, stream reassembly dropped speed and inference_geo, and the pass-through handler never read speed off the request body, so fast-mode spend was logged at the standard rate. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 2 +- .../streaming_chunk_builder_utils.py | 16 +++ .../anthropic_passthrough_logging_handler.py | 24 +++- .../streaming_chunk_builder_utils.py | 2 + .../test_litellm_logging.py | 35 ++++++ .../test_streaming_chunk_builder_utils.py | 39 ++++++ ...t_anthropic_passthrough_logging_handler.py | 113 ++++++++++++++++++ 7 files changed, 228 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c2dc7189934..15330a2a910 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3278,7 +3278,7 @@ class Logging(LiteLLMLoggingBaseClass): model=self.model, messages=[], logging_obj=self, - optional_params={}, + optional_params=self.optional_params or {}, api_key="", request_data={}, encoding=litellm.encoding, diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index d52d9849310..5ceb64547a0 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -569,6 +569,8 @@ class ChunkProcessor: # lost and 1h cache writes get billed at the 5m rate. cache_creation_token_details: Optional[CacheCreationTokenDetails] = None cost: Optional[float] = None + inference_geo: Optional[str] = None + speed: Optional[str] = None for chunk in chunks: usage_chunk = self._extract_usage_chunk(chunk) @@ -627,6 +629,13 @@ class ChunkProcessor: if usage_chunk_dict["cost"] is not None: cost = usage_chunk_dict["cost"] + chunk_inference_geo = getattr(usage_chunk, "inference_geo", None) + if isinstance(chunk_inference_geo, str): + inference_geo = chunk_inference_geo + chunk_speed = getattr(usage_chunk, "speed", None) + if isinstance(chunk_speed, str): + speed = chunk_speed + prompt_tokens_details = self._attach_cache_creation_token_details( prompt_tokens_details, cache_creation_token_details ) @@ -647,6 +656,8 @@ class ChunkProcessor: completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, cost=cost, + inference_geo=inference_geo, + speed=speed, ) @staticmethod @@ -806,6 +817,11 @@ class ChunkProcessor: if cost is not None: setattr(returned_usage, "cost", cost) + if calculated_usage_per_chunk["inference_geo"] is not None: + setattr(returned_usage, "inference_geo", calculated_usage_per_chunk["inference_geo"]) + if calculated_usage_per_chunk["speed"] is not None: + setattr(returned_usage, "speed", calculated_usage_per_chunk["speed"]) + # Return a new usage object with the new values returned_usage = Usage(**returned_usage.model_dump()) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 50e90699194..112780581de 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -80,7 +80,9 @@ class AnthropicPassthroughLoggingHandler: model=model, messages=[], logging_obj=logging_obj, - optional_params={}, + optional_params=AnthropicPassthroughLoggingHandler._cost_relevant_request_params( + request_body or kwargs.get("request_body") + ), api_key="", request_data={}, encoding=litellm.encoding, @@ -102,6 +104,15 @@ class AnthropicPassthroughLoggingHandler: "kwargs": kwargs, } + @staticmethod + def _cost_relevant_request_params(request_body: Optional[dict]) -> dict: + """ + Request params that change how the response is priced, and so must reach the + usage-building paths. Anthropic's ``speed=fast`` multiplies non-cache token cost. + """ + speed = (request_body or {}).get("speed") + return {"speed": speed} if isinstance(speed, str) else {} + @staticmethod def _get_user_from_metadata( passthrough_logging_payload: PassthroughStandardLoggingPayload, @@ -315,6 +326,7 @@ class AnthropicPassthroughLoggingHandler: - Logs in litellm callbacks """ + speed = AnthropicPassthroughLoggingHandler._cost_relevant_request_params(request_body).get("speed") model = request_body.get("model", "") # Check if it's available in the logging object if ( @@ -334,6 +346,7 @@ class AnthropicPassthroughLoggingHandler: all_chunks=all_chunks, litellm_logging_obj=litellm_logging_obj, model=model, + speed=speed, ) except Exception as e: # stream_chunk_builder re-raises assembly failures (as litellm.APIError) @@ -355,6 +368,7 @@ class AnthropicPassthroughLoggingHandler: complete_streaming_response = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( all_chunks=all_chunks, model=model, + speed=speed, ) except Exception as e: verbose_proxy_logger.warning( @@ -419,6 +433,7 @@ class AnthropicPassthroughLoggingHandler: all_chunks: Sequence[Union[str, bytes]], litellm_logging_obj: LiteLLMLoggingObj, model: str, + speed: Optional[str] = None, ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: """ Builds complete response from raw Anthropic chunks. @@ -443,11 +458,13 @@ class AnthropicPassthroughLoggingHandler: all_chunks=collapsed, litellm_logging_obj=litellm_logging_obj, model=model, + speed=speed, ) return AnthropicPassthroughLoggingHandler._build_complete_streaming_response_legacy( all_chunks=all_chunks, litellm_logging_obj=litellm_logging_obj, model=model, + speed=speed, ) # Anthropic SSE block/delta types that the fast path is NOT allowed to @@ -575,6 +592,7 @@ class AnthropicPassthroughLoggingHandler: all_chunks: Sequence[Union[str, bytes]], litellm_logging_obj: LiteLLMLoggingObj, model: str, + speed: Optional[str] = None, ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: """ Original reconstruction: convert every SSE event to a generic chunk @@ -590,6 +608,7 @@ class AnthropicPassthroughLoggingHandler: anthropic_model_response_iterator = AnthropicModelResponseIterator( streaming_response=None, sync_stream=False, + speed=speed, ) all_openai_chunks = [] @@ -649,6 +668,7 @@ class AnthropicPassthroughLoggingHandler: def _build_usage_only_response_from_chunks( all_chunks: Sequence[Union[str, bytes]], model: str, + speed: Optional[str] = None, ) -> Optional[ModelResponse]: """ Build a usage-bearing ModelResponse from Anthropic SSE token-usage events, for @@ -742,7 +762,7 @@ class AnthropicPassthroughLoggingHandler: usage_object["server_tool_use"] = _server_tool_use if inference_geo is not None: usage_object["inference_geo"] = inference_geo - usage_obj = AnthropicConfig().calculate_usage(usage_object=usage_object, reasoning_content=None) + usage_obj = AnthropicConfig().calculate_usage(usage_object=usage_object, reasoning_content=None, speed=speed) return ModelResponse( model=resolved_model, choices=[ diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index c9f9d4e6baa..4ff47171e32 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -15,3 +15,5 @@ class UsagePerChunk(TypedDict): completion_tokens_details: Optional[CompletionTokensDetails] prompt_tokens_details: Optional[PromptTokensDetailsWrapper] cost: Optional[float] + inference_geo: Optional[str] + speed: Optional[str] diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index eaa4bd3e3fc..1cd4174c57c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4153,3 +4153,38 @@ def test_pre_call_does_not_pin_request_in_module_state(logging_obj): logging_obj.post_call(original_response='{"ok": true}', input=big_input, api_key="sk-test") assert litellm.error_logs == {} + + +def test_handle_anthropic_messages_response_logging_preserves_fast_mode_speed(): + """/v1/messages non-streaming rebuilds usage by re-transforming the raw Anthropic + response. Anthropic's fast-mode multiplier is applied off ``usage.speed``, which the + response body never carries, so the request's optional params have to be passed in or + fast-mode spend is logged at the standard rate.""" + import httpx + + logging_obj = LitellmLogging( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="lit-5115", + function_id="lit-5115", + ) + logging_obj.optional_params = {"speed": "fast"} + logging_obj.model_call_details["httpx_response"] = httpx.Response( + status_code=200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1000, "cache_read_input_tokens": 200, "output_tokens": 100}, + }, + ) + + result = logging_obj._handle_anthropic_messages_response_logging(result=None) + + assert result.usage.speed == "fast" # type: ignore[attr-defined] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index be8c5a05601..72bbad7ae49 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -992,3 +992,42 @@ def test_cost_field_in_usage_chunks(): assert usage.cost == 0.00025 assert usage.prompt_tokens == 10 assert usage.completion_tokens == 5 + + +def test_anthropic_speed_and_geo_survive_stream_assembly(): + """Anthropic prices fast mode and non-global regions with a multiplier read off + ``usage.speed`` / ``usage.inference_geo``. Dropping them while reassembling a stream + bills streamed fast-mode calls at the standard rate.""" + from litellm.llms.anthropic.cost_calculation import cost_per_token + + def _usage(**extra): + usage = Usage(completion_tokens=100, prompt_tokens=1000, total_tokens=1100) + for key, value in extra.items(): + setattr(usage, key, value) + return usage + + def _chunk(usage): + return ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="claude-opus-4-8", + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="Hi"))], + usage=usage, + ) + + fast_chunk = _chunk(_usage(speed="fast", inference_geo="global")) + fast_usage = ChunkProcessor(chunks=[fast_chunk]).calculate_usage( + chunks=[fast_chunk], model="claude-opus-4-8", completion_output="Hi" + ) + standard_chunk = _chunk(_usage(inference_geo="global")) + standard_usage = ChunkProcessor(chunks=[standard_chunk]).calculate_usage( + chunks=[standard_chunk], model="claude-opus-4-8", completion_output="Hi" + ) + + assert fast_usage.speed == "fast" + assert fast_usage.inference_geo == "global" + assert getattr(standard_usage, "speed", None) is None + + fast_cost = sum(cost_per_token(model="claude-opus-4-8", usage=fast_usage)) + standard_cost = sum(cost_per_token(model="claude-opus-4-8", usage=standard_usage)) + assert fast_cost == pytest.approx(standard_cost * 2.0) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 947a7a64beb..ffbdb3485ae 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2192,3 +2192,116 @@ class TestAnthropicResponseCostRecordedOnModelCallDetails: logging_obj.model_call_details["response_cost"] == kwargs["response_cost"] ) assert logging_obj.model_call_details["response_cost"] > 0 + + +class TestAnthropicPassthroughFastMode: + """Anthropic charges a provider-specific multiplier for ``speed=fast``, and the + multiplier is applied off ``usage.speed``. The pass-through handler only sees the + speed in the request body, so it has to thread it into every usage-building path or + fast-mode pass-through spend is under-reported.""" + + MODEL = "claude-opus-4-8" + STREAM_CHUNKS = [ + 'event: message_start', + 'data: {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant",' + ' "model": "claude-opus-4-8", "content": [], "stop_reason": null,' + ' "usage": {"input_tokens": 1000, "cache_read_input_tokens": 200, "output_tokens": 0}}}', + 'event: content_block_start', + 'data: {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}', + 'event: content_block_delta', + 'data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}', + 'event: content_block_stop', + 'data: {"type": "content_block_stop", "index": 0}', + 'event: message_delta', + 'data: {"type": "message_delta", "delta": {"stop_reason": "end_turn"},' + ' "usage": {"input_tokens": 1000, "cache_read_input_tokens": 200, "output_tokens": 100}}', + 'event: message_stop', + 'data: {"type": "message_stop"}', + ] + + def _logging_obj(self) -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model=self.MODEL, + messages=[], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="fast-mode", + function_id="fast-mode", + ) + + def _cost(self, response) -> float: + import litellm + + return litellm.completion_cost(completion_response=response, model=f"anthropic/{self.MODEL}") + + def _expected_fast_cost(self, standard_cost: float) -> float: + import litellm + + model_info = litellm.get_model_info(model=self.MODEL, custom_llm_provider="anthropic") + cache_read_cost = 200 * (model_info.get("cache_read_input_token_cost") or 0.0) + return (standard_cost - cache_read_cost) * 2.0 + cache_read_cost + + def test_non_streaming_applies_fast_multiplier(self): + import httpx + + response_body = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": self.MODEL, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1000, "cache_read_input_tokens": 200, "output_tokens": 100}, + } + + def _handle(request_body): + logging_obj = self._logging_obj() + logging_obj.model_call_details["stream"] = False + return AnthropicPassthroughLoggingHandler.anthropic_passthrough_handler( + httpx_response=httpx.Response(status_code=200, json=response_body), + response_body=response_body, + logging_obj=logging_obj, + url_route="https://api.anthropic.com/v1/messages", + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body=request_body, + ) + + fast = _handle({"model": self.MODEL, "speed": "fast"}) + standard = _handle({"model": self.MODEL}) + + assert fast["result"].usage.speed == "fast" + assert self._cost(fast["result"]) == pytest.approx(self._expected_fast_cost(self._cost(standard["result"]))) + + def test_streaming_reconstruction_applies_fast_multiplier(self): + fast = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=self.STREAM_CHUNKS, + litellm_logging_obj=self._logging_obj(), + model=self.MODEL, + speed="fast", + ) + standard = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=self.STREAM_CHUNKS, + litellm_logging_obj=self._logging_obj(), + model=self.MODEL, + ) + + assert fast.usage.speed == "fast" + assert self._cost(fast) == pytest.approx(self._expected_fast_cost(self._cost(standard))) + + def test_usage_only_fallback_applies_fast_multiplier(self): + fast = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=self.STREAM_CHUNKS, + model=self.MODEL, + speed="fast", + ) + standard = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=self.STREAM_CHUNKS, + model=self.MODEL, + ) + + assert fast.usage.speed == "fast" + assert self._cost(fast) == pytest.approx(self._expected_fast_cost(self._cost(standard))) From 833670f7dbe32331f8689171bc9c0310ea11dd0a Mon Sep 17 00:00:00 2001 From: elinacse Date: Sun, 2 Aug 2026 12:20:46 +0530 Subject: [PATCH 031/672] fix(batch): track cost for managed batches with no attributable key/user/team LiteLLM_ManagedObjectTable only stores created_by (user_id) and team_id, never the raw API key hash. A batch created with the master key or a team-less key has both null, so CheckBatchCost's synthetic logging_obj for the completed batch carried no attributable key/user/team/end-user. _should_track_cost_callback silently skipped the DB write in that case (by design, to avoid tracking truly anonymous requests), with no error or warning: batch_processed still became true, but no LiteLLM_SpendLogs row was ever written despite real, already-incurred provider cost. Extend the same allowance already made for unauthenticated pass-through requests to aretrieve_batch's cost event, and pass job.team_id through so a batch's team gets real attribution when one exists. --- .../proxy/common_utils/check_batch_cost.py | 1 + .../proxy/hooks/proxy_track_cost_callback.py | 12 +- .../proxy_unit_tests/test_check_batch_cost.py | 128 ++++++++++++++++++ .../hooks/test_proxy_track_cost_callback.py | 17 ++- 4 files changed, 154 insertions(+), 4 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 22f9f40ecd8..0214c6cceb6 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -502,6 +502,7 @@ class CheckBatchCost: }, "metadata": { "user_api_key_user_id": creator_user_id, + "user_api_key_team_id": getattr(job, "team_id", None), **user_info, }, }, diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 857429fa89f..2ff7808868d 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -34,11 +34,17 @@ from litellm.types.utils import ( ) from litellm.utils import get_end_user_id_for_cost_tracking -_PASS_THROUGH_CALL_TYPES: frozenset[str] = frozenset( +_UNATTRIBUTED_TRACKABLE_CALL_TYPES: frozenset[str] = frozenset( { CallTypes.pass_through.value, CallTypes.llm_passthrough_route.value, CallTypes.allm_passthrough_route.value, + # CheckBatchCost's synthetic logging_obj for a completed managed batch only ever + # carries user_api_key_user_id (from LiteLLM_ManagedObjectTable.created_by) and + # user_api_key_team_id (from .team_id) -- both are None for batches created with + # the master key or a team-less key, since the table never stores the raw key + # hash. The batch already incurred real provider cost, so track it regardless. + CallTypes.aretrieve_batch.value, } ) @@ -434,6 +440,8 @@ def _should_track_cost_callback( the request with no key/user/team/end-user to attribute spend to. Those requests still forward real provider traffic that operators expect to see in request/usage logs, so they are tracked even when unauthenticated. + The same reasoning applies to a completed managed batch's cost event + (see _UNATTRIBUTED_TRACKABLE_CALL_TYPES). """ # don't run track cost callback if user opted into disabling spend @@ -442,7 +450,7 @@ def _should_track_cost_callback( if user_api_key is not None or user_id is not None or team_id is not None or end_user_id is not None: return True - return call_type in _PASS_THROUGH_CALL_TYPES + return call_type in _UNATTRIBUTED_TRACKABLE_CALL_TYPES def _get_budget_reservation_from_metadata(metadata: dict) -> dict | None: diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index a15abd023d8..42499f2ac55 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -420,6 +420,134 @@ class TestCheckBatchCost: ), "update() must include batch_processed=True when column is present" assert update_data["status"] == "complete" + @pytest.mark.asyncio + async def test_completed_batch_with_no_attributable_owner_still_writes_spend_log( + self, check_batch_cost_instance, mock_prisma_client, mock_llm_router + ): + """Regression: a batch created with the master key or a team-less key has + created_by=None and team_id=None on LiteLLM_ManagedObjectTable (the table + never stores the raw key hash). CheckBatchCost's synthetic logging_obj for + such a batch then carries no attributable key/user/team/end-user, and + before the fix _should_track_cost_callback silently skipped the DB write + with no error or warning: batch_processed still became True, but no + LiteLLM_SpendLogs row was ever written. + + Unlike the other tests in this file, this one does NOT mock + litellm_logging.Logging or async_success_handler -- it runs the real + logging pipeline through to _ProxyDBLogger, which is the exact gap that + let the original bug ship undetected. + """ + import litellm + from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger + + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=0) + mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + + mock_job = MagicMock() + mock_job.id = "job-unattributed-1" + mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA==" + mock_job.created_by = None + mock_job.team_id = None + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job]) + + # A real LiteLLMBatch (not a bare MagicMock): this test runs the real + # litellm_logging.Logging pipeline, which type-checks the result via + # isinstance(..., LiteLLMBatch) before it will compute/attach a cost. + from litellm.types.utils import LiteLLMBatch + + mock_response = LiteLLMBatch( + id="batch-1", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-input-123", + object="batch", + status="completed", + output_file_id="file-output-123", + ) + + mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) + mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"}) + + mock_deployment = MagicMock() + mock_deployment.litellm_params.custom_llm_provider = "openai" + mock_deployment.litellm_params.model = "gpt-4" + mock_deployment.model_info.model_dump.return_value = {} + mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment) + + mock_file_content = MagicMock() + mock_file_content.content = b'{"id":"req-1"}' + + decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;" + + db_logger = _ProxyDBLogger() + mock_update_database = AsyncMock() + + # Unlike the other tests in this file, this one runs the real + # litellm_logging.Logging pipeline, which calls + # _is_base64_encoded_unified_file_id an extra time (checking result.id + # after it's reset to job.unified_object_id). Key off the argument + # instead of a fixed-length side_effect list so the exact call count + # doesn't matter. + def _fake_is_base64_encoded(file_id): + return decoded_id if file_id == mock_job.unified_object_id else None + + with ( + patch( + "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id", + side_effect=_fake_is_base64_encoded, + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id", + return_value="model-123", + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id", + return_value="batch-456", + ), + patch( + "litellm.files.main.afile_content", + new_callable=AsyncMock, + return_value=mock_file_content, + ), + patch( + "litellm.batches.batch_utils._get_file_content_as_dictionary", + return_value=[{"id": "req-1"}], + ), + patch( + "litellm.batches.batch_utils.calculate_batch_cost_and_usage", + new_callable=AsyncMock, + return_value=( + 0.01, + {"prompt_tokens": 10, "completion_tokens": 5}, + ["gpt-4"], + ), + ), + patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("gpt-4", "openai", None, None), + ), + patch.object(litellm, "_async_success_callback", [db_logger]), + patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + MagicMock( + db_spend_update_writer=MagicMock(update_database=mock_update_database), + slack_alerting_instance=MagicMock(customer_spend_alert=AsyncMock()), + ), + ), + patch("litellm.proxy.proxy_server.increment_spend_counters", AsyncMock()), + patch("litellm.proxy.proxy_server.update_cache", AsyncMock()), + ): + await check_batch_cost_instance.check_batch_cost() + + mock_update_database.assert_awaited_once() + assert mock_update_database.call_args.kwargs["response_cost"] == 0.01 + assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, ( + "the job must still be marked processed once cost tracking succeeds" + ) + @pytest.mark.asyncio async def test_cost_tracking_failure_leaves_job_unprocessed( self, check_batch_cost_instance, mock_prisma_client, mock_llm_router diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index f289148101a..69f04ce2bbe 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1186,6 +1186,7 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata(): ("pass_through_endpoint", True), ("llm_passthrough_route", True), ("allm_passthrough_route", True), + ("aretrieve_batch", True), ("acompletion", False), ("call_mcp_tool", False), (None, False), @@ -1194,7 +1195,14 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata(): def test_should_track_cost_callback_pass_through_without_owner(call_type, expected): """Regression for LIT-3782: unauthenticated pass-through requests (auth=false) carry no key/user/team/end-user, yet must still be tracked so they land in - LiteLLM_SpendLogs. Other call types with no owner stay untracked.""" + LiteLLM_SpendLogs. Other call types with no owner stay untracked. + + aretrieve_batch is included for the same reason: CheckBatchCost's synthetic + logging_obj for a completed managed batch only ever carries + user_api_key_user_id/user_api_key_team_id from LiteLLM_ManagedObjectTable, + both of which are None for a batch created with the master key or a + team-less key (the table never stores the raw key hash). Before this fix, + such a batch's cost silently never reached LiteLLM_SpendLogs.""" assert ( _should_track_cost_callback( user_api_key=None, @@ -1211,6 +1219,7 @@ def test_should_track_cost_callback_pass_through_without_owner(call_type, expect "call_type, expect_spend_log", [ ("pass_through_endpoint", True), + ("aretrieve_batch", True), ("acompletion", False), (None, False), ], @@ -1223,7 +1232,11 @@ async def test_track_cost_callback_logs_unauthenticated_pass_through_request( cost callback with no key/user/team/end-user. Before the fix the spend-log write was skipped and the request never appeared in request/usage logs. It must now be written for pass-through call types while other unauthenticated - calls remain skipped.""" + calls remain skipped. + + aretrieve_batch is included because CheckBatchCost's completed-batch cost + event reaches this same callback with no attributable key/user/team when + the batch was created with the master key or a team-less key.""" logger = _ProxyDBLogger() kwargs = { From ef614b7b5bcdf94472b876bea64ad17e5dfd4282 Mon Sep 17 00:00:00 2001 From: milan Date: Mon, 3 Aug 2026 14:35:47 +0000 Subject: [PATCH 032/672] refactor(vertex_ai): keep the batch embeddings translation within the LIT002 ceiling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/vertex_ai/files/transformation.py | 118 +++++++++++------- 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 9363540fe1b..bbb97a1edc2 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -74,11 +74,11 @@ _GCP_LABEL_VALUE_MAX_LEN = 63 _CUSTOM_ID_RAW_LABEL_PREFIX = "b32_" _VERTEX_BATCH_KEY_FIELD = "key" _MANAGED_GCS_MODEL_PATH_PATTERN = re.compile(r"publishers/[^/]+/models/([^/?]+)") -_EMBED_REQUEST_FIELD_BY_GEMINI_PARAM = { - "outputDimensionality": "output_dimensionality", - "taskType": "task_type", - "title": "title", -} +_EMBED_REQUEST_FIELD_BY_GEMINI_PARAM = ( + ("outputDimensionality", "output_dimensionality"), + ("taskType", "task_type"), + ("title", "title"), +) _VERTEX_BATCH_FANNED_OUT_KEY_PATTERN = re.compile(r"(?P[^#]*)#(?P\d+)/(?P\d+)") @@ -153,12 +153,15 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, Any]) -> str: key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) if key is not None: return unquote(str(key)) - request_data = vertex_output_row.get("request") or {} - return _get_litellm_batch_custom_id_from_labels(request_data.get("labels") or {}) + request_data = vertex_output_row.get("request") + labels = request_data.get("labels") if isinstance(request_data, Mapping) else None + return _get_litellm_batch_custom_id_from_labels(labels) -def _get_litellm_batch_custom_id_from_labels(labels: dict[str, Any]) -> str: +def _get_litellm_batch_custom_id_from_labels(labels: Mapping[str, Any] | None) -> str: """Prefer encoded custom_id when present (see _set_litellm_batch_custom_id_labels).""" + if not labels: + return "unknown" raw = labels.get("litellm_custom_id_raw") if raw: raw_chunks = [str(raw)] @@ -195,7 +198,8 @@ def _is_vertex_embeddings_batch_output_row(vertex_output_row: Mapping[str, Any]) def _openai_batch_output_row( custom_id: str, body: Mapping[str, Any] | None = None, - error: Mapping[str, str] | None = None, + error_code: str | None = None, + error_message: str = "", ) -> Mapping[str, Any]: """ One row of an OpenAI batch output file. Per the OpenAI Batch spec, failed rows set @@ -211,7 +215,7 @@ def _openai_batch_output_row( "request_id": body.get("id", ""), "body": body, }, - "error": error, + "error": None if error_code is None else {"code": error_code, "message": error_message}, } @@ -233,6 +237,19 @@ def _split_vertex_batch_key(vertex_output_row: Mapping[str, Any]) -> tuple[str, return unquote(match["custom_id"]), int(match["index"]) +def _embedding_prompt_token_count(vertex_response: Mapping[str, Any]) -> int: + """ + Prompt tokens billed for one Vertex Gemini Embedding batch row. + + Live rows report usage under `usageMetadata`; the documented `tokenCount` is kept as + a fallback. + """ + usage_metadata = vertex_response.get("usageMetadata") + if isinstance(usage_metadata, Mapping): + return int(usage_metadata.get("promptTokenCount") or 0) + return int(vertex_response.get("tokenCount") or 0) + + def _vertex_embeddings_rows_to_openai_batch_output_row( custom_id: str, vertex_output_rows: tuple[Mapping[str, Any], ...], @@ -247,23 +264,19 @@ def _vertex_embeddings_rows_to_openai_batch_output_row( An entry that asked for several embeddings at once maps to several rows here, which become the indexed elements of a single `data` array. One failed element fails the - whole entry, since an OpenAI batch row is either a response or an error. Live rows - report usage under `usageMetadata`; the documented `tokenCount` is kept as a - fallback. Rows carry no `modelVersion`, so the model comes from the batch they - belong to. + whole entry, since an OpenAI batch row is either a response or an error. Rows carry + no `modelVersion`, so the model comes from the batch they belong to. """ status = next((row["status"] for row in vertex_output_rows if row.get("status")), "") if status: return _openai_batch_output_row( custom_id=custom_id, - error={"code": "vertex_ai_error", "message": status}, + error_code="vertex_ai_error", + error_message=status, ) - responses = tuple(row.get("response") or {} for row in vertex_output_rows) - token_count = sum( - int((response.get("usageMetadata") or {}).get("promptTokenCount") or response.get("tokenCount") or 0) - for response in responses - ) + responses = tuple(row["response"] for row in vertex_output_rows) + token_count = sum(_embedding_prompt_token_count(response) for response in responses) body = EmbeddingResponse( model=model or "", data=[ @@ -361,6 +374,27 @@ def _vertex_batch_embeddings_key(custom_id: str, index: int, total: int) -> str: return encoded_custom_id if total < 2 else f"{encoded_custom_id}#{index}/{total}" +def _vertex_embeddings_row(key: str | None, embed_content_request: Mapping[str, Any]) -> Mapping[str, Any]: + """ + One Vertex Gemini Embedding batch input row. + + The config fields live inside the `EmbedContentRequest` under their snake_case batch + names, and the OpenAI `custom_id` rides along in the top-level `key` that Vertex + echoes back. + """ + request = { + "content": embed_content_request["content"], + **{ + request_field: embed_content_request[gemini_param] + for gemini_param, request_field in _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM + if gemini_param in embed_content_request + }, + } + if key is None: + return {"request": request} + return {_VERTEX_BATCH_KEY_FIELD: key, "request": request} + + def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( openai_entry: Mapping[str, Any], ) -> tuple[Mapping[str, Any], ...]: @@ -381,7 +415,9 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( API Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/batch-prediction-genai-embeddings """ - openai_request_body = openai_entry.get("body") or {} + openai_request_body = openai_entry.get("body") + if not isinstance(openai_request_body, dict): + raise ValueError("`body` is required on /v1/embeddings batch requests, but was not provided") embedding_input = openai_request_body.get("input") if embedding_input is None: raise ValueError("`input` is required on /v1/embeddings batch requests, but was not provided") @@ -400,27 +436,16 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) custom_id = openai_entry.get("custom_id") return tuple( - { - **( - {} - if custom_id is None - else { - _VERTEX_BATCH_KEY_FIELD: _vertex_batch_embeddings_key( - custom_id=str(custom_id), - index=index, - total=len(embed_content_requests), - ) - } + _vertex_embeddings_row( + key=None + if custom_id is None + else _vertex_batch_embeddings_key( + custom_id=str(custom_id), + index=index, + total=len(embed_content_requests), ), - "request": { - "content": embed_content_request["content"], - **{ - request_field: embed_content_request[gemini_param] - for gemini_param, request_field in _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM.items() - if gemini_param in embed_content_request - }, - }, - } + embed_content_request=embed_content_request, + ) for index, embed_content_request in enumerate(embed_content_requests) ) @@ -1008,7 +1033,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): request=httpx.Request(method="POST", url="https://example.com"), ) - all_lines = itertools.chain([first_line], lines) + all_lines = itertools.chain((first_line,), lines) # Embedding rows are grouped by `custom_id` rather than transformed one at a # time, since an entry that asked for several embeddings comes back as @@ -1064,7 +1089,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): if has_error: return _openai_batch_output_row( custom_id=custom_id, - error={"code": "vertex_ai_error", "message": status}, + error_code="vertex_ai_error", + error_message=status, ) # Transform successful response using existing transformation @@ -1096,8 +1122,6 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): except Exception as e: return _openai_batch_output_row( custom_id=custom_id, - error={ - "code": "transformation_error", - "message": f"Failed to transform response: {e!s}", - }, + error_code="transformation_error", + error_message=f"Failed to transform response: {e!s}", ) From a7250f4eeab560299215773b50ba32405f597a1c Mon Sep 17 00:00:00 2001 From: milan Date: Mon, 3 Aug 2026 17:11:50 +0000 Subject: [PATCH 033/672] fix(bedrock): normalize /v1/completions and /v1/responses batch records Bedrock managed-batch file upload read `messages` unconditionally, so a JSONL record shaped for /v1/completions (`prompt`) or /v1/responses (`input`) reached the per-provider transform with an empty message list. Anthropic and Nova rejected it at POST /v1/files, and the passthrough providers shipped an empty conversation to AWS. Classify each record by its OpenAI batch `url`, then normalize the non-embedding shapes to chat completions before the Bedrock transforms: `prompt` wraps into user messages the way litellm.text_completion does in real time, and `input` goes through the existing Responses-to-Chat bridge. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 22 +- litellm/llms/bedrock/files/transformation.py | 194 ++++++++-- litellm/types/llms/bedrock.py | 15 + ...ore_utils_prompt_templates_common_utils.py | 43 +++ .../test_bedrock_files_transformation.py | 347 ++++++++++++++++-- 5 files changed, 559 insertions(+), 62 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 639c93dfb80..777ba398d5a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,7 @@ import io import json import mimetypes import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from os import PathLike from pathlib import Path from typing import ( @@ -1742,3 +1742,23 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]: idx = end_idx return results + + +def text_completion_prompt_to_messages(prompt: str | Sequence[str]) -> tuple[AllMessageValues, ...]: + """ + Wrap an OpenAI ``/v1/completions`` ``prompt`` into Chat Completion messages. + + Mirrors what ``litellm.text_completion`` does on the real-time path: a + string becomes a single user message, and a list of strings becomes one + user message per element. Pre-tokenized prompts (``list[int]`` / + ``list[list[int]]``) are only meaningful for the OpenAI-family text + endpoints, so they are rejected here rather than silently forwarded, as is + an empty prompt, which every chat-shaped provider rejects downstream. + """ + if isinstance(prompt, str) and prompt: + return (ChatCompletionUserMessage(role="user", content=prompt),) + if isinstance(prompt, Sequence) and prompt and all(isinstance(entry, str) and entry for entry in prompt): + return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in prompt) + raise ValueError( + f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {type(prompt).__name__}." + ) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 3656088cb9d..0aa832780e5 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -2,7 +2,9 @@ import base64 import json import os import time -from collections.abc import Mapping, MutableMapping +from collections.abc import Iterable, Mapping, MutableMapping +from functools import cache +from itertools import chain from types import MappingProxyType from typing import ( Any, @@ -12,7 +14,7 @@ from urllib.parse import unquote import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -28,12 +30,16 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + text_completion_prompt_to_messages, +) from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, LiteLLMLoggingObj, ) +from litellm.types.llms.bedrock import BedrockBatchRecordKind from litellm.types.llms.openai import ( AllMessageValues, CreateFileRequest, @@ -43,6 +49,8 @@ from litellm.types.llms.openai import ( OpenAICreateFileRequestOptionalParams, OpenAIFileObject, PathLike, + ResponseInputParam, + ResponsesAPIOptionalRequestParams, ) from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums from litellm.utils import get_llm_provider @@ -57,6 +65,26 @@ from ..common_utils import BedrockError S3_SIGNED_GET_HEADERS_PARAM = "_s3_signed_get_headers" +def _frozen_mapping(items: Iterable[tuple[str, Any]]) -> Mapping[str, Any]: + return MappingProxyType(dict(items)) + + +# JSONL batch records are untyped json, so the `/v1/responses` fields are +# validated into their concrete Responses API types before being handed to the +# Responses-to-Chat bridge. Both adapters drop keys the Responses API doesn't +# define, which is what the bridge would ignore anyway. Built on first use +# rather than at import: `ResponseInputParam` is a deep union and only batch +# files carrying `/v1/responses` records need it. +@cache +def _responses_input_adapter() -> TypeAdapter[str | ResponseInputParam]: + return TypeAdapter(str | ResponseInputParam) + + +@cache +def _responses_request_adapter() -> TypeAdapter[ResponsesAPIOptionalRequestParams]: + return TypeAdapter(ResponsesAPIOptionalRequestParams) + + class _BedrockS3RequestParams(BaseModel): """Typed view of the credential/region params the S3 GetObject path reads.""" @@ -305,41 +333,55 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # example; add others here as they adopt the same schema. CONVERSE_INVOKE_PROVIDERS = ("nova",) - # OpenAI batch URL that signals an embedding request. Per OpenAI Batch API - # spec, every JSONL record carries a `url` field; we use it as the - # authoritative signal to route the line to the embedding code path - # instead of inferring from the presence of `input` vs `messages`. + # OpenAI batch URLs that select which request shape a JSONL line carries. + # Per the OpenAI Batch API spec every record carries a `url`, so we use it + # as the authoritative routing signal instead of inferring from the + # presence of `input` vs `prompt` vs `messages`. OPENAI_EMBEDDINGS_URL = "/v1/embeddings" + OPENAI_TEXT_COMPLETIONS_URL = "/v1/completions" + OPENAI_RESPONSES_URL = "/v1/responses" @staticmethod - def _is_embedding_record(openai_jsonl_record: dict[str, Any]) -> bool: + def _classify_batch_record(openai_jsonl_record: Mapping[str, Any]) -> BedrockBatchRecordKind: """ - Decide whether an OpenAI batch JSONL line is an embedding request. + Decide which OpenAI endpoint shape an OpenAI batch JSONL line carries. - Precedence (strict - any explicit `url` short-circuits): - 1. `url == "/v1/embeddings"` -> embedding. Authoritative per the - OpenAI Batch API spec. - 2. Any other non-empty `url` (e.g. `/v1/chat/completions`) -> NOT - embedding. We trust the caller's explicit signal even if the - body would otherwise suggest embedding; misrouting a chat - record into the embedding transformer would corrupt the - modelInput, while a chat-shaped body sent to the chat path - either succeeds or fails cleanly inside that transformer. - 3. `url` missing/empty -> fall back to body shape. Requires - `input` present AND `messages` absent so a malformed record - carrying both keys routes to the chat path (safer default: - Anthropic transforms ignore unknown top-level keys, whereas - the embedding transformer would silently drop the messages). + Precedence (strict - any recognized `url` short-circuits): + 1. A `url` matching a supported endpoint wins. Authoritative per the + OpenAI Batch API spec, which requires it on every record. + 2. Any other non-empty `url` -> chat. We trust the caller's explicit + signal rather than re-deriving it from the body, and an + unexpectedly-shaped body fails cleanly inside the chat + transformer instead of being silently misrouted. + 3. `url` missing/empty -> fall back to body shape. `messages` wins + over the other keys so a malformed record carrying several of + them keeps its conversation instead of having it dropped, and a + bare `input` stays an embedding for backwards compatibility + (that ambiguity with `/v1/responses` is only resolvable from + `url`). """ - url = openai_jsonl_record.get("url") - if url == BedrockFilesConfig.OPENAI_EMBEDDINGS_URL: - return True - if url: - return False - body = openai_jsonl_record.get("body", {}) - if not isinstance(body, dict): - return False - return "input" in body and "messages" not in body + match openai_jsonl_record.get("url"): + case BedrockFilesConfig.OPENAI_EMBEDDINGS_URL: + return BedrockBatchRecordKind.EMBEDDING + case BedrockFilesConfig.OPENAI_TEXT_COMPLETIONS_URL: + return BedrockBatchRecordKind.TEXT_COMPLETION + case BedrockFilesConfig.OPENAI_RESPONSES_URL: + return BedrockBatchRecordKind.RESPONSES + case None | "": + pass + case _: + return BedrockBatchRecordKind.CHAT + + body = openai_jsonl_record.get("body") + if not isinstance(body, Mapping): + return BedrockBatchRecordKind.CHAT + if "messages" in body: + return BedrockBatchRecordKind.CHAT + if "prompt" in body: + return BedrockBatchRecordKind.TEXT_COMPLETION + if "input" in body: + return BedrockBatchRecordKind.EMBEDDING + return BedrockBatchRecordKind.CHAT # Identifier for the Bedrock Titan v2 InvokeModel body schema as stored # in `model_prices_and_context_window.json`. Centralized so future @@ -546,9 +588,83 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) return dict(titan_config._transform_request(input=input_text, inference_params=inference_params)) + @staticmethod + def _transform_text_completion_body_to_chat_body(openai_request_body: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Rewrite an OpenAI `/v1/completions` batch body as a Chat Completions body. + + Bedrock batch `modelInput` is the model's InvokeModel/Converse body, and + no Bedrock batch model takes a bare `prompt`, so the wrapping that + `litellm.text_completion` does in real time has to happen here too. + """ + prompt = openai_request_body.get("prompt") + if prompt is None: + raise ValueError( + "Batch record for /v1/completions is missing required `prompt` field: " + f"model={openai_request_body.get('model', '')}" + ) + return _frozen_mapping( + chain( + ((key, value) for key, value in openai_request_body.items() if key != "prompt"), + (("messages", text_completion_prompt_to_messages(prompt)),), + ) + ) + + @staticmethod + def _transform_responses_body_to_chat_body(openai_request_body: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Rewrite an OpenAI `/v1/responses` batch body as a Chat Completions body. + + Delegates to the same Responses-to-Chat bridge the real-time path uses + for providers without a native Responses API (which is every Bedrock + model), so `input`, `instructions`, `max_output_tokens` and the tool + params translate identically in batch and real time. The bridge always + emits a `tools` key; an empty one is dropped rather than shipped as an + empty array inside `modelInput`. + """ + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + responses_input = openai_request_body.get("input") + if responses_input is None: + raise ValueError( + "Batch record for /v1/responses is missing required `input` field: " + f"model={openai_request_body.get('model', '')}" + ) + chat_body = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=openai_request_body.get("model", ""), + input=_responses_input_adapter().validate_python(responses_input), + responses_api_request=_responses_request_adapter().validate_python( + _frozen_mapping( + (key, value) for key, value in openai_request_body.items() if key not in ("model", "input") + ) + ), + ) + return _frozen_mapping((key, value) for key, value in chat_body.items() if key != "tools" or value) + + @staticmethod + def _transform_batch_body_to_chat_body( + openai_request_body: Mapping[str, Any], + record_kind: BedrockBatchRecordKind, + ) -> Mapping[str, Any]: + """ + Normalize a non-embedding batch body to the Chat Completions shape the + per-provider Bedrock transformations expect. + """ + match record_kind: + case BedrockBatchRecordKind.TEXT_COMPLETION: + return BedrockFilesConfig._transform_text_completion_body_to_chat_body(openai_request_body) + case BedrockBatchRecordKind.RESPONSES: + return BedrockFilesConfig._transform_responses_body_to_chat_body(openai_request_body) + case BedrockBatchRecordKind.CHAT: + return openai_request_body + case BedrockBatchRecordKind.EMBEDDING: + raise ValueError("Embedding batch records do not have a chat-completion equivalent") + def _map_openai_to_bedrock_params( self, - openai_request_body: dict[str, Any], + openai_request_body: Mapping[str, Any], provider: str | None = None, ) -> dict[str, Any]: """ @@ -659,14 +775,18 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): provider = self.get_bedrock_invoke_provider(model) # Route to the embedding transformer when the OpenAI batch line - # targets /v1/embeddings; otherwise fall back to the existing - # chat-completion path. We branch here (rather than inside + # targets /v1/embeddings; every other endpoint shape is normalized + # to chat completions first. We branch here (rather than inside # `_map_openai_to_bedrock_params`) so the chat helper keeps its # narrow contract and the embedding helper can evolve independently. - if self._is_embedding_record(_openai_jsonl_content): + record_kind = self._classify_batch_record(_openai_jsonl_content) + if record_kind is BedrockBatchRecordKind.EMBEDDING: model_input = self._map_openai_embedding_to_bedrock_params(openai_request_body=openai_body) else: - model_input = self._map_openai_to_bedrock_params(openai_request_body=openai_body, provider=provider) + model_input = self._map_openai_to_bedrock_params( + openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind), + provider=provider, + ) # Create Bedrock batch record record_id = _openai_jsonl_content.get("custom_id", f"CALL{str(idx).zfill(7)}") diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index d9f8229dbed..f7a4682cb03 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1,4 +1,5 @@ import json +from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union from typing_extensions import TYPE_CHECKING, Required, TypedDict, override @@ -1100,3 +1101,17 @@ class BedrockInvokeAnthropicMessagesRequest(TypedDict, total=False): # supported subset and strips the field entirely when nothing remains, so # other edit types (e.g. `clear_thinking_20251015`) never reach Bedrock. context_management: dict + + +class BedrockBatchRecordKind(Enum): + """ + Which OpenAI endpoint shape a line of a Bedrock managed-batch JSONL file + carries. Bedrock batch `modelInput` is always the model's InvokeModel / + Converse body, so every non-embedding shape is normalized to Chat + Completions before being handed to the per-provider transformation. + """ + + CHAT = "chat" + TEXT_COMPLETION = "text_completion" + RESPONSES = "responses" + EMBEDDING = "embedding" diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 1b1db634ed2..d10ccf77703 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -721,3 +721,46 @@ class TestUnpackLegacyDefs: out = unpack_legacy_defs(schema) assert "components" not in out assert out["properties"]["r0"]["properties"]["p0"] == {"type": "string"} + + +class TestTextCompletionPromptToMessages: + """`/v1/completions` prompt wrapping, shared by the real-time and batch paths.""" + + def test_string_prompt_becomes_single_user_message(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + text_completion_prompt_to_messages, + ) + + assert text_completion_prompt_to_messages("summarize this") == ( + {"role": "user", "content": "summarize this"}, + ) + + def test_list_of_strings_becomes_one_message_each(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + text_completion_prompt_to_messages, + ) + + assert text_completion_prompt_to_messages(["first", "second"]) == ( + {"role": "user", "content": "first"}, + {"role": "user", "content": "second"}, + ) + + @pytest.mark.parametrize( + "prompt", + [ + [1, 2, 3], + [[1, 2], [3, 4]], + ["ok", 7], + [], + "", + None, + {"role": "user"}, + ], + ) + def test_unsupported_prompt_shapes_raise(self, prompt): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + text_completion_prompt_to_messages, + ) + + with pytest.raises(ValueError, match="non-empty string or a non-empty list of strings"): + text_completion_prompt_to_messages(prompt) diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index c548fe53e15..87b03b02e1a 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -1072,21 +1072,34 @@ class TestBedrockFilesEmbeddingTransformation: is None ) - def test_is_embedding_record_helper(self): - """Helper detects embeddings via `url` first, then by body shape.""" + def test_classify_batch_record_helper(self): + """Helper classifies by `url` first, then by body shape.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + from litellm.types.llms.bedrock import BedrockBatchRecordKind - assert BedrockFilesConfig._is_embedding_record( - {"url": "/v1/embeddings", "body": {"input": "x"}} + assert ( + BedrockFilesConfig._classify_batch_record( + {"url": "/v1/embeddings", "body": {"input": "x"}} + ) + is BedrockBatchRecordKind.EMBEDDING ) # body-only fallback - assert BedrockFilesConfig._is_embedding_record({"body": {"input": "x"}}) - # chat shape - assert not BedrockFilesConfig._is_embedding_record( - {"url": "/v1/chat/completions", "body": {"messages": []}} + assert ( + BedrockFilesConfig._classify_batch_record({"body": {"input": "x"}}) + is BedrockBatchRecordKind.EMBEDDING + ) + # chat shape + assert ( + BedrockFilesConfig._classify_batch_record( + {"url": "/v1/chat/completions", "body": {"messages": []}} + ) + is BedrockBatchRecordKind.CHAT + ) + # ambiguous body without any recognized key is treated as chat + assert ( + BedrockFilesConfig._classify_batch_record({"body": {}}) + is BedrockBatchRecordKind.CHAT ) - # ambiguous body without `input` is treated as not-embedding - assert not BedrockFilesConfig._is_embedding_record({"body": {}}) def test_explicit_chat_url_with_input_body_short_circuits_to_chat(self): """Explicit url=/v1/chat/completions wins even if body looks like embedding. @@ -1097,15 +1110,20 @@ class TestBedrockFilesEmbeddingTransformation: """ from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + from litellm.types.llms.bedrock import BedrockBatchRecordKind + # Direct helper assertion - assert not BedrockFilesConfig._is_embedding_record( - { - "url": "/v1/chat/completions", - "body": { - "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - "input": "this would mis-route under the old precedence", - }, - } + assert ( + BedrockFilesConfig._classify_batch_record( + { + "url": "/v1/chat/completions", + "body": { + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "input": "this would mis-route under the old precedence", + }, + } + ) + is BedrockBatchRecordKind.CHAT ) # End-to-end: a record like this routes through the chat path. We @@ -1164,20 +1182,301 @@ class TestBedrockFilesEmbeddingTransformation: with pytest.raises(ValueError, match="must be a string"): BedrockFilesConfig._coerce_embedding_input_to_string({"unsupported": True}) - def test_other_non_embedding_urls_route_to_chat(self): - """Any non-/v1/embeddings url short-circuits to chat path.""" + def test_other_non_embedding_urls_do_not_route_to_embeddings(self): + """An `input` body only means "embedding" when the url says so.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + from litellm.types.llms.bedrock import BedrockBatchRecordKind # /v1/completions (legacy completions endpoint) - assert not BedrockFilesConfig._is_embedding_record( - {"url": "/v1/completions", "body": {"input": "x"}} + assert ( + BedrockFilesConfig._classify_batch_record( + {"url": "/v1/completions", "body": {"input": "x"}} + ) + is BedrockBatchRecordKind.TEXT_COMPLETION + ) + assert ( + BedrockFilesConfig._classify_batch_record( + {"url": "/v1/responses", "body": {"input": "x"}} + ) + is BedrockBatchRecordKind.RESPONSES ) # Arbitrary unknown url - caller's explicit signal still wins - assert not BedrockFilesConfig._is_embedding_record( - {"url": "/v1/responses", "body": {"input": "x"}} + assert ( + BedrockFilesConfig._classify_batch_record( + {"url": "/v1/moderations", "body": {"input": "x"}} + ) + is BedrockBatchRecordKind.CHAT ) +class TestBedrockBatchNonChatEndpointRecords: + """`/v1/completions` and `/v1/responses` JSONL records (issue #35639). + + Bedrock batch `modelInput` is always the model's InvokeModel/Converse body, + so a record shaped for another OpenAI endpoint has to be normalized to chat + completions first. Before this normalization every record below either + raised `BadRequestError` at `POST /v1/files` (Anthropic, Nova) or silently + shipped an empty `messages` list to AWS (passthrough providers). + """ + + ANTHROPIC_MODEL = "bedrock/us.anthropic.claude-sonnet-4-6" + NOVA_MODEL = "bedrock/us.amazon.nova-pro-v1:0" + PASSTHROUGH_MODEL = "bedrock/openai.gpt-oss-120b-1:0" + + def _transform(self, record: dict) -> dict: + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + result = BedrockFilesConfig()._transform_openai_jsonl_content_to_bedrock_jsonl_content([record]) + assert len(result) == 1 + assert result[0]["recordId"] == record["custom_id"] + return result[0]["modelInput"] + + def test_anthropic_text_completion_record_wraps_prompt(self): + model_input = self._transform( + { + "custom_id": "1", + "method": "POST", + "url": "/v1/completions", + "body": { + "model": self.ANTHROPIC_MODEL, + "prompt": "Summarize the following call transcript", + "max_tokens": 64, + }, + } + ) + + assert model_input == { + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "Summarize the following call transcript"}], + } + ], + "max_tokens": 64, + "anthropic_version": "bedrock-2023-05-31", + } + + def test_anthropic_text_completion_record_keeps_every_prompt_in_a_list(self): + model_input = self._transform( + { + "custom_id": "2", + "method": "POST", + "url": "/v1/completions", + "body": { + "model": self.ANTHROPIC_MODEL, + "prompt": ["first prompt", "second prompt"], + "max_tokens": 8, + }, + } + ) + + # Consecutive user messages are merged by the Anthropic transform, the + # same way they are on the real-time path. + assert model_input["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "first prompt"}, + {"type": "text", "text": "second prompt"}, + ], + } + ] + assert "prompt" not in model_input + + def test_anthropic_responses_record_wraps_string_input(self): + model_input = self._transform( + { + "custom_id": "3", + "method": "POST", + "url": "/v1/responses", + "body": { + "model": self.ANTHROPIC_MODEL, + "input": "hi", + "max_output_tokens": 16, + }, + } + ) + + assert model_input == { + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "max_tokens": 16, + "anthropic_version": "bedrock-2023-05-31", + } + assert "tools" not in model_input, "an empty tools array must not be shipped to Bedrock" + + def test_anthropic_responses_record_maps_instructions_and_input_items(self): + """The Responses-specific params go through the same bridge as real time.""" + model_input = self._transform( + { + "custom_id": "4", + "method": "POST", + "url": "/v1/responses", + "body": { + "model": self.ANTHROPIC_MODEL, + "instructions": "be terse", + "input": [ + {"role": "user", "content": "what is 2+2?"}, + {"role": "assistant", "content": "4"}, + {"role": "user", "content": "and 3+3?"}, + ], + "max_output_tokens": 32, + "temperature": 0.2, + }, + } + ) + + assert model_input["system"] == [{"type": "text", "text": "be terse"}] + assert model_input["max_tokens"] == 32 + assert model_input["temperature"] == 0.2 + assert [message["role"] for message in model_input["messages"]] == [ + "user", + "assistant", + "user", + ] + assert model_input["messages"][-1]["content"] == [{"type": "text", "text": "and 3+3?"}] + assert "input" not in model_input + assert "max_output_tokens" not in model_input + + @pytest.mark.parametrize( + "body", + [ + {"prompt": "hi"}, + {"input": "hi"}, + ], + ids=["prompt", "input"], + ) + def test_nova_converse_record_wraps_prompt_and_input(self, body): + url = "/v1/completions" if "prompt" in body else "/v1/responses" + model_input = self._transform( + { + "custom_id": "5", + "method": "POST", + "url": url, + "body": {"model": self.NOVA_MODEL, **body}, + } + ) + + assert model_input["messages"] == [{"role": "user", "content": [{"text": "hi"}]}] + + @pytest.mark.parametrize( + "body", + [ + {"prompt": "hi"}, + {"input": "hi"}, + ], + ids=["prompt", "input"], + ) + def test_passthrough_provider_record_no_longer_emits_empty_messages(self, body): + """The passthrough branch used to emit `{"messages": [], "prompt": ...}`. + + That shape is accepted by `POST /v1/files`, so the whole batch job was + submitted to AWS and only failed there. + """ + url = "/v1/completions" if "prompt" in body else "/v1/responses" + model_input = self._transform( + { + "custom_id": "6", + "method": "POST", + "url": url, + "body": {"model": self.PASSTHROUGH_MODEL, **body}, + } + ) + + # Asserted on the serialized form, since the passthrough branch hands + # `messages` straight to S3 without a per-provider transform. + assert json.loads(json.dumps(model_input)) == {"messages": [{"role": "user", "content": "hi"}]} + + def test_mixed_endpoints_in_one_file_keep_their_own_shapes(self): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + result = BedrockFilesConfig()._transform_openai_jsonl_content_to_bedrock_jsonl_content( + [ + { + "custom_id": "chat", + "url": "/v1/chat/completions", + "body": { + "model": self.ANTHROPIC_MODEL, + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 4, + }, + }, + { + "custom_id": "text", + "url": "/v1/completions", + "body": {"model": self.ANTHROPIC_MODEL, "prompt": "hi", "max_tokens": 4}, + }, + { + "custom_id": "responses", + "url": "/v1/responses", + "body": {"model": self.ANTHROPIC_MODEL, "input": "hi", "max_output_tokens": 4}, + }, + { + "custom_id": "embedding", + "url": "/v1/embeddings", + "body": {"model": "bedrock/amazon.titan-embed-text-v2:0", "input": "hi"}, + }, + ] + ) + + assert [record["recordId"] for record in result] == [ + "chat", + "text", + "responses", + "embedding", + ] + for record in result[:3]: + assert record["modelInput"]["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "hi"}]} + ] + assert result[3]["modelInput"] == {"inputText": "hi"} + + @pytest.mark.parametrize( + ("url", "expected_message"), + [ + ("/v1/completions", "missing required `prompt` field"), + ("/v1/responses", "missing required `input` field"), + ], + ) + def test_missing_required_field_raises_actionable_error(self, url, expected_message): + with pytest.raises(ValueError, match=expected_message): + self._transform( + { + "custom_id": "7", + "method": "POST", + "url": url, + "body": {"model": self.ANTHROPIC_MODEL, "max_tokens": 4}, + } + ) + + def test_prompt_body_without_url_is_still_wrapped(self): + """A record can omit `url`; the body shape then decides.""" + model_input = self._transform( + { + "custom_id": "8", + "body": {"model": self.ANTHROPIC_MODEL, "prompt": "hi", "max_tokens": 4}, + } + ) + + assert model_input["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + def test_messages_win_over_prompt_when_url_is_absent(self): + model_input = self._transform( + { + "custom_id": "9", + "body": { + "model": self.ANTHROPIC_MODEL, + "messages": [{"role": "user", "content": "from messages"}], + "prompt": "from prompt", + "max_tokens": 4, + }, + } + ) + + assert model_input["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "from messages"}]} + ] + + class TestBedrockFileContentTransformation: """SigV4-signed S3 GetObject retrieval of Bedrock batch output files.""" From 6def61e672d95c4b145613009cd3064d0a133475 Mon Sep 17 00:00:00 2001 From: milan Date: Mon, 3 Aug 2026 17:22:46 +0000 Subject: [PATCH 034/672] fix(bedrock): keep /v1/responses batch metadata through the chat bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/files/transformation.py | 1 + .../files/test_bedrock_files_transformation.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 0aa832780e5..baa2630556a 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -640,6 +640,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): (key, value) for key, value in openai_request_body.items() if key not in ("model", "input") ) ), + metadata=openai_request_body.get("metadata"), ) return _frozen_mapping((key, value) for key, value in chat_body.items() if key != "tools" or value) diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index 87b03b02e1a..09aa6b1cf2d 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -1337,6 +1337,23 @@ class TestBedrockBatchNonChatEndpointRecords: assert "input" not in model_input assert "max_output_tokens" not in model_input + def test_responses_record_keeps_metadata(self): + """`metadata` reaches the bridge, which reads it as its own kwarg.""" + model_input = self._transform( + { + "custom_id": "4b", + "method": "POST", + "url": "/v1/responses", + "body": { + "model": self.PASSTHROUGH_MODEL, + "input": "hi", + "metadata": {"tenant": "acct-1"}, + }, + } + ) + + assert model_input["metadata"] == {"tenant": "acct-1"} + @pytest.mark.parametrize( "body", [ From 46751ad83ed3307fdc0e966d7e850660df2446d9 Mon Sep 17 00:00:00 2001 From: Darien Kindlund Date: Tue, 4 Aug 2026 16:17:01 -0400 Subject: [PATCH 035/672] fix(anthropic): coerce explicit additionalProperties to false in output_format schema Anthropic's structured outputs reject any `additionalProperties` value other than `false` ("output_format.schema: For 'object' type, 'additionalProperties: true' is not supported. Please set 'additionalProperties' to false") `filter_anthropic_output_schema` only added the key when it was absent, so an explicit `true` (or a sub-schema) was copied verbatim into output_format.schema and 400'd. Coerce it for object schemas instead, at every recursion depth, matching what the Anthropic Python/TypeScript SDKs do The permissive tool-use path (map_response_format_to_anthropic_tool, used for vertex_ai) is deliberately left alone Fixes #35808 --- litellm/llms/anthropic/chat/transformation.py | 2 +- .../anthropic/test_anthropic_schema_filter.py | 72 ++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 51b862e79d9..19f5174579b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -597,7 +597,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Anthropic requires additionalProperties=false for object schemas # See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs - if result.get("type") == "object" and "additionalProperties" not in result: + if result.get("type") == "object": result["additionalProperties"] = False return result diff --git a/tests/litellm/llms/anthropic/test_anthropic_schema_filter.py b/tests/litellm/llms/anthropic/test_anthropic_schema_filter.py index c10ac5532a0..71c9cfe8f41 100644 --- a/tests/litellm/llms/anthropic/test_anthropic_schema_filter.py +++ b/tests/litellm/llms/anthropic/test_anthropic_schema_filter.py @@ -281,7 +281,10 @@ class TestFilterAnthropicOutputSchema: "unevaluatedProperties", ): assert field not in result - assert 'properties whose names match each pattern must satisfy: {"^x": {"type": "string"}}' in result["description"] + assert ( + 'properties whose names match each pattern must satisfy: {"^x": {"type": "string"}}' + in result["description"] + ) assert 'property names must satisfy: {"pattern": "^[a-z]+$"}' in result["description"] assert 'dependent required properties: {"first": ["last"]}' in result["description"] assert 'dependent schemas: {"first": {"required": ["last"]}}' in result["description"] @@ -347,3 +350,70 @@ class TestFilterAnthropicOutputSchema: "all array items must be unique, minimum number of matching items: 2, " "maximum number of matching items: 3." ) + + def test_coerces_explicit_additional_properties_true(self): + """An explicit ``additionalProperties: true`` must be coerced to false. + + Anthropic rejects anything other than false with: + "output_format.schema: For 'object' type, 'additionalProperties: true' is + not supported". + """ + schema = { + "type": "object", + "additionalProperties": True, + "properties": {"a": {"type": "string"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["additionalProperties"] is False + + def test_coerces_additional_properties_true_when_nested(self): + """Nested object schemas are coerced too, at every recursion site.""" + schema = { + "type": "object", + "properties": { + "obj": { + "type": "object", + "additionalProperties": True, + "properties": {"a": {"type": "string"}}, + }, + "rows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": True, + "properties": {"b": {"type": "string"}}, + }, + }, + }, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["properties"]["obj"]["additionalProperties"] is False + assert result["properties"]["rows"]["items"]["additionalProperties"] is False + + def test_coerces_additional_properties_sub_schema(self): + """A sub-schema value (free-form map) is also rejected by Anthropic.""" + schema = { + "type": "object", + "additionalProperties": {"type": "string"}, + "properties": {"a": {"type": "string"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["additionalProperties"] is False + + def test_explicit_additional_properties_false_is_preserved(self): + """The already-correct value must survive untouched.""" + schema = { + "type": "object", + "additionalProperties": False, + "properties": {"a": {"type": "string"}}, + } + + result = AnthropicConfig.filter_anthropic_output_schema(schema) + + assert result["additionalProperties"] is False From 0c0e1e8374d7e956e65d275ebf5f2f832ec374b9 Mon Sep 17 00:00:00 2001 From: Miles Adkins Date: Wed, 5 Aug 2026 13:21:15 -0500 Subject: [PATCH 036/672] feat(fireworks_ai): translate NIM/vLLM extra params to Fireworks-native args Requests migrated from NIM/vLLM servers carry extras that flow through the extra_body passthrough verbatim, but the Fireworks chat completions API either names them differently or does not accept them at all. Add FireworksAIConfig.map_extra_body_params, invoked from the fireworks chat dispatch, which renames truncate_prompt_tokens to prompt_truncate_len, maps chat_template_kwargs.enable_thinking to reasoning_effort, converts guided_json/guided_grammar/guided_choice to response_format, and drops the remaining extras (min_tokens, stop_token_ids, skip_special_tokens, guided_regex, etc.) with a debug log. Alias and competing-constraint combinations raise BadRequestError. Unrecognized extras keep passing through untouched, as do fireworks-native params like top_k. --- .../llms/fireworks_ai/chat/transformation.py | 162 +++++++++++- litellm/main.py | 6 +- .../test_fireworks_ai_chat_transformation.py | 249 ++++++++++++++++++ 3 files changed, 415 insertions(+), 2 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index a796aa47b70..4740e84d513 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -1,5 +1,5 @@ import json -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping from typing import Any, Final, Literal, cast import httpx @@ -61,6 +61,36 @@ def _extract_fireworks_hidden_params(payload: dict) -> dict: return {**top_level, **per_choice} +def _json_schema_response_format(schema: object) -> Mapping[str, object]: + return {"type": "json_schema", "json_schema": {"schema": schema}} # mutable-ok: JSON request body + + +_NIM_VLLM_STRIP_PARAMS: Final = frozenset( + { + "min_tokens", + "stop_token_ids", + "include_stop_str_in_output", + "skip_special_tokens", + "spaces_between_special_tokens", + "best_of", + "use_beam_search", + "guided_decoding_backend", + "guided_regex", + "add_generation_prompt", + "continue_final_message", + "add_special_tokens", + "detokenize", + "allowed_token_ids", + "bad_words", + } +) + +_EXTRA_BODY_CONSUMED_PARAMS: Final = ( + frozenset({"truncate_prompt_tokens", "chat_template_kwargs", "guided_json", "guided_grammar", "guided_choice"}) + | _NIM_VLLM_STRIP_PARAMS +) + + class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): """ Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions @@ -273,6 +303,136 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return optional_params + def map_extra_body_params(self, optional_params: Mapping[str, object], model: str) -> dict: # noqa: LIT001 # http handler pops extra_body off the returned dict + extra_body: Final = optional_params.get("extra_body") + if not isinstance(extra_body, dict): + return dict(optional_params) # mutable-ok: JSON request body + + self._validate_extra_body_conflicts(extra_body=extra_body, optional_params=optional_params, model=model) + stripped: Final = tuple(sorted(k for k in extra_body if k in _NIM_VLLM_STRIP_PARAMS)) + if stripped: + verbose_logger.debug( + "fireworks_ai does not support NIM/vLLM params %s for model=%s; dropping them from the request.", + stripped, + model, + ) + promoted: Final = ( + *self._translate_truncate_prompt_tokens(extra_body), + *self._translate_chat_template_kwargs(extra_body, model), + *self._translate_guided_params(extra_body), + ) + remaining: Final = tuple((k, v) for k, v in extra_body.items() if k not in _EXTRA_BODY_CONSUMED_PARAMS) + base: Final = {k: v for k, v in optional_params.items() if k != "extra_body"} # mutable-ok: JSON request body + return { # mutable-ok: JSON request body + **base, + **dict(promoted), # mutable-ok: JSON request body + **({"extra_body": dict(remaining)} if remaining else {}), # mutable-ok: JSON request body + } + + def _validate_extra_body_conflicts( + self, extra_body: Mapping[str, object], optional_params: Mapping[str, object], model: str + ) -> None: + if "truncate_prompt_tokens" in extra_body and ( + "prompt_truncate_len" in extra_body or "prompt_truncate_len" in optional_params + ): + raise litellm.BadRequestError( + message=( + "Fireworks AI chat completions received both `truncate_prompt_tokens` and " + "`prompt_truncate_len`; they are aliases, send only one." + ), + model=model, + llm_provider="fireworks_ai", + ) + chat_template_kwargs: Final = extra_body.get("chat_template_kwargs") + if ( + isinstance(chat_template_kwargs, dict) + and "enable_thinking" in chat_template_kwargs + and ("reasoning_effort" in optional_params or "thinking" in optional_params) + ): + raise litellm.BadRequestError( + message=( + "Fireworks AI chat completions does not support specifying both " + "`chat_template_kwargs.enable_thinking` and `reasoning_effort`/`thinking` in the same request." + ), + model=model, + llm_provider="fireworks_ai", + ) + guided_params: Final = tuple( + k for k in ("guided_json", "guided_grammar", "guided_choice") if extra_body.get(k) is not None + ) + if len(guided_params) > 1: + raise litellm.BadRequestError( + message=( + f"Fireworks AI chat completions received multiple guided decoding params " + f"{guided_params}; send only one." + ), + model=model, + llm_provider="fireworks_ai", + ) + if guided_params and "response_format" in optional_params: + raise litellm.BadRequestError( + message=( + f"Fireworks AI chat completions received both `{guided_params[0]}` and " + "`response_format`; they are competing output constraints, send only one." + ), + model=model, + llm_provider="fireworks_ai", + ) + + @staticmethod + def _translate_truncate_prompt_tokens(extra_body: Mapping[str, object]) -> tuple[tuple[str, object], ...]: + if extra_body.get("truncate_prompt_tokens") is None: + return () + return (("prompt_truncate_len", extra_body["truncate_prompt_tokens"]),) + + def _translate_chat_template_kwargs( + self, extra_body: Mapping[str, object], model: str + ) -> tuple[tuple[str, object], ...]: + chat_template_kwargs: Final = extra_body.get("chat_template_kwargs") + if chat_template_kwargs is None: + return () + if not isinstance(chat_template_kwargs, dict): + raise litellm.BadRequestError( + message="Fireworks AI chat completions expects `chat_template_kwargs` to be an object.", + model=model, + llm_provider="fireworks_ai", + ) + other_keys: Final = tuple(sorted(k for k in chat_template_kwargs if k != "enable_thinking")) + if other_keys: + verbose_logger.debug( + "fireworks_ai does not support chat_template_kwargs keys %s for model=%s; dropping them.", + other_keys, + model, + ) + if "enable_thinking" not in chat_template_kwargs: + return () + if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): + verbose_logger.debug( + "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs.enable_thinking.", + model, + ) + return () + effort: Final = "medium" if chat_template_kwargs["enable_thinking"] else "none" + return (("reasoning_effort", effort),) + + @staticmethod + def _translate_guided_params(extra_body: Mapping[str, object]) -> tuple[tuple[str, object], ...]: + if extra_body.get("guided_json") is not None: + return (("response_format", _json_schema_response_format(extra_body["guided_json"])),) + if extra_body.get("guided_grammar") is not None: + grammar_response_format: Final = { # mutable-ok: JSON request body + "type": "grammar", + "grammar": extra_body["guided_grammar"], + } + return (("response_format", grammar_response_format),) + if extra_body.get("guided_choice") is not None: + choice_schema: Final = { # mutable-ok: JSON request body + "type": "string", + "enum": extra_body["guided_choice"], + } + return (("response_format", _json_schema_response_format(choice_schema)),) + return () + def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]: for tool in tools: if tool.get("type") != "function": diff --git a/litellm/main.py b/litellm/main.py index f906c78f9ae..660e024b113 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1711,11 +1711,15 @@ def _complete_fireworks_ai( messages: Final = ctx.messages model: Final = ctx.model model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params provider_config: Final = ctx.provider_config shared_session: Final = ctx.shared_session stream: Final = ctx.stream timeout: Final = ctx.timeout + optional_params: Final = ( + provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) + if isinstance(provider_config, litellm.FireworksAIConfig) + else ctx.optional_params + ) try: response: Final = base_llm_http_handler.completion( diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 94945ed4bfb..3fbbc70916a 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1282,3 +1282,252 @@ def test_streaming_surfaces_fireworks_response_fields(): assert surfaced["fireworks_raw_outputs"] == [raw_output] assert surfaced["fireworks_perf_metrics"] == {"prompt-tokens": 5} assert surfaced["fireworks_prompt_token_ids"] == [1, 2, 3] + + +def test_map_extra_body_params_translates_truncate_prompt_tokens(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL + ) + assert result == {"prompt_truncate_len": 4096} + + +def test_map_extra_body_params_truncate_prompt_tokens_conflicts_with_alias(): + config = FireworksAIConfig() + with pytest.raises(litellm.BadRequestError, match="aliases"): + config.map_extra_body_params( + {"prompt_truncate_len": 2048, "extra_body": {"truncate_prompt_tokens": 4096}}, + _REASONING_MODEL, + ) + with pytest.raises(litellm.BadRequestError, match="aliases"): + config.map_extra_body_params( + {"extra_body": {"truncate_prompt_tokens": 4096, "prompt_truncate_len": 2048}}, + _REASONING_MODEL, + ) + + +def test_map_extra_body_params_chat_template_kwargs_enable_thinking(): + config = FireworksAIConfig() + disabled = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _REASONING_MODEL, + ) + assert disabled == {"reasoning_effort": "none"} + + enabled = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": True}}}, + _REASONING_MODEL, + ) + assert enabled == {"reasoning_effort": "medium"} + + +def test_map_extra_body_params_chat_template_kwargs_conflicts_with_reasoning_effort(): + config = FireworksAIConfig() + with pytest.raises(litellm.BadRequestError, match="enable_thinking"): + config.map_extra_body_params( + { + "reasoning_effort": "high", + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + }, + _REASONING_MODEL, + ) + + +def test_map_extra_body_params_chat_template_kwargs_conflicts_with_thinking(): + config = FireworksAIConfig() + with pytest.raises(litellm.BadRequestError, match="enable_thinking"): + config.map_extra_body_params( + { + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "extra_body": {"chat_template_kwargs": {"enable_thinking": True}}, + }, + _REASONING_MODEL, + ) + + +def test_map_extra_body_params_chat_template_kwargs_dropped_for_non_reasoning_model(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False, "custom_flag": 1}}}, + _NON_REASONING_MODEL, + ) + assert result == {} + + +def test_map_extra_body_params_guided_json(): + config = FireworksAIConfig() + schema = {"type": "object", "properties": {"x": {"type": "string"}}} + result = config.map_extra_body_params( + {"extra_body": {"guided_json": schema}}, _REASONING_MODEL + ) + assert result == { + "response_format": {"type": "json_schema", "json_schema": {"schema": schema}} + } + + +def test_map_extra_body_params_guided_grammar_and_choice(): + config = FireworksAIConfig() + grammar = config.map_extra_body_params( + {"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL + ) + assert grammar == { + "response_format": {"type": "grammar", "grammar": "root ::= 'hello'"} + } + + choice = config.map_extra_body_params( + {"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL + ) + assert choice == { + "response_format": { + "type": "json_schema", + "json_schema": {"schema": {"type": "string", "enum": ["yes", "no"]}}, + } + } + + +def test_map_extra_body_params_guided_conflicts_with_response_format(): + config = FireworksAIConfig() + with pytest.raises(litellm.BadRequestError, match="response_format"): + config.map_extra_body_params( + { + "response_format": {"type": "json_object"}, + "extra_body": {"guided_json": {"type": "object"}}, + }, + _REASONING_MODEL, + ) + + +def test_map_extra_body_params_multiple_guided_params_rejected(): + config = FireworksAIConfig() + with pytest.raises(litellm.BadRequestError, match="multiple guided decoding params"): + config.map_extra_body_params( + {"extra_body": {"guided_json": {"type": "object"}, "guided_grammar": "root ::= 'x'"}}, + _REASONING_MODEL, + ) + + +@pytest.mark.parametrize( + "param,value", + [ + ("min_tokens", 10), + ("stop_token_ids", [1, 2]), + ("include_stop_str_in_output", True), + ("skip_special_tokens", False), + ("spaces_between_special_tokens", True), + ("best_of", 2), + ("use_beam_search", True), + ("guided_decoding_backend", "outlines"), + ("guided_regex", "[0-9]+"), + ("add_generation_prompt", True), + ("continue_final_message", True), + ("add_special_tokens", False), + ("detokenize", True), + ("allowed_token_ids", [1]), + ("bad_words", ["foo"]), + ], +) +def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value, caplog): + import logging + + config = FireworksAIConfig() + with caplog.at_level(logging.DEBUG): + result = config.map_extra_body_params( + {"extra_body": {param: value}}, _REASONING_MODEL + ) + assert result == {} + assert param in caplog.text + + +def test_map_extra_body_params_preserves_unknown_passthrough(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"top_k": 40, "some_future_param": "x", "truncate_prompt_tokens": 100}}, + _REASONING_MODEL, + ) + assert result == { + "prompt_truncate_len": 100, + "extra_body": {"top_k": 40, "some_future_param": "x"}, + } + + +def test_map_extra_body_params_no_extra_body(): + config = FireworksAIConfig() + assert config.map_extra_body_params({}, _REASONING_MODEL) == {} + unchanged = {"temperature": 0.5, "extra_body": None} + assert config.map_extra_body_params(unchanged, _REASONING_MODEL) == unchanged + + +def test_nim_vllm_extras_translated_end_to_end_in_request_body(): + """ + Passing NIM/vLLM extras to litellm.completion must reach the Fireworks + request body translated, not verbatim: truncate_prompt_tokens becomes + prompt_truncate_len, chat_template_kwargs.enable_thinking becomes + reasoning_effort, min_tokens is dropped, and fireworks-native top_k still + passes through. Asserts on the actual JSON posted to the API, so a revert + of the _complete_fireworks_ai wiring fails this test. + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + model = "accounts/fireworks/models/glm-5p1" + body = { + "id": "chat-1", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + raw_response = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + raw_response.text = json.dumps(body) + raw_response.json = lambda: body + + client = HTTPHandler() + with patch.object(client, "post", return_value=raw_response) as mock_post: + litellm.completion( + model=f"fireworks_ai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + truncate_prompt_tokens=4096, + chat_template_kwargs={"enable_thinking": False}, + min_tokens=10, + top_k=40, + ) + + request_body = json.loads(mock_post.call_args.kwargs["data"]) + assert request_body["prompt_truncate_len"] == 4096 + assert "truncate_prompt_tokens" not in request_body + assert request_body["reasoning_effort"] == "none" + assert "chat_template_kwargs" not in request_body + assert "min_tokens" not in request_body + assert request_body["top_k"] == 40 + + +def test_in_schema_unsupported_params_still_raise(): + """ + The extras translation channel does not weaken the supported-params gate + for in-schema OpenAI params: store is still rejected with drop_params=False + and dropped with drop_params=True. + """ + with pytest.raises(litellm.UnsupportedParamsError): + litellm.get_optional_params( + model="accounts/fireworks/models/llama-v3-70b-instruct", + custom_llm_provider="fireworks_ai", + drop_params=False, + store=True, + ) + optional_params = litellm.get_optional_params( + model="accounts/fireworks/models/llama-v3-70b-instruct", + custom_llm_provider="fireworks_ai", + drop_params=True, + store=True, + ) + assert "store" not in optional_params From 599283584f2d16448c25a1ee4fbfdda2062eecf3 Mon Sep 17 00:00:00 2001 From: Miles Adkins Date: Wed, 5 Aug 2026 15:09:16 -0500 Subject: [PATCH 037/672] feat(fireworks_ai): drop reasoning_effort=auto to the model default Fireworks rejects reasoning_effort="auto" (accepted set: low, medium, high, xhigh, max, none, adaptive), so OpenAI-compatible clients sending it 400. Omitting the param means model default on Fireworks, which is exactly what auto means on OpenAI's side, so skip it in map_openai_params instead of forwarding. --- litellm/llms/fireworks_ai/chat/transformation.py | 6 ++++-- .../test_fireworks_ai_chat_transformation.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 4740e84d513..a05b160413e 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -295,7 +295,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): optional_params["reasoning_effort"] = "medium" elif value is False: optional_params["reasoning_effort"] = "none" - else: + elif value != "auto": optional_params["reasoning_effort"] = value elif param in supported_openai_params: if value is not None: @@ -303,7 +303,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return optional_params - def map_extra_body_params(self, optional_params: Mapping[str, object], model: str) -> dict: # noqa: LIT001 # http handler pops extra_body off the returned dict + def map_extra_body_params( + self, optional_params: Mapping[str, object], model: str + ) -> dict: # mutable-ok: http handler pops extra_body off the returned dict extra_body: Final = optional_params.get("extra_body") if not isinstance(extra_body, dict): return dict(optional_params) # mutable-ok: JSON request body diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 3fbbc70916a..bbb7fb197d0 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1153,6 +1153,22 @@ def test_reasoning_effort_integer_passthrough(): assert isinstance(result["reasoning_effort"], int) +def test_reasoning_effort_auto_dropped_to_model_default(): + """ + Fireworks rejects reasoning_effort="auto" (accepted set: low/medium/high/ + xhigh/max/none/adaptive). Omitting the param is the model default, which is + exactly what "auto" means on OpenAI's side, so it must not reach the request. + """ + config = FireworksAIConfig() + result = config.map_openai_params( + {"reasoning_effort": "auto"}, + {}, + _REASONING_MODEL, + drop_params=False, + ) + assert "reasoning_effort" not in result + + def test_transform_response_captures_perf_metrics(): body = { **_BASE_CHAT_COMPLETION_RESPONSE, From 6d80d0509976feb702aad744cb7aff5fa81d3f54 Mon Sep 17 00:00:00 2001 From: Miles Adkins Date: Wed, 5 Aug 2026 15:56:33 -0500 Subject: [PATCH 038/672] fix(fireworks_ai): align extras translation with the API gateway matrix min_tokens is accepted natively by the Fireworks API (verified live), so stop stripping it and let it pass through extra_body. Add the NIM-specific include_reasoning and nvext keys to the strip set. enable_thinking=true now omits reasoning_effort (model default) instead of forcing medium, matching the gateway translation and preserving default-off models' behavior; enable_thinking=false still maps to none. --- litellm/llms/fireworks_ai/chat/transformation.py | 8 +++++--- .../test_fireworks_ai_chat_transformation.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index a05b160413e..b5c82129f45 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -67,7 +67,6 @@ def _json_schema_response_format(schema: object) -> Mapping[str, object]: _NIM_VLLM_STRIP_PARAMS: Final = frozenset( { - "min_tokens", "stop_token_ids", "include_stop_str_in_output", "skip_special_tokens", @@ -82,6 +81,8 @@ _NIM_VLLM_STRIP_PARAMS: Final = frozenset( "detokenize", "allowed_token_ids", "bad_words", + "include_reasoning", + "nvext", } ) @@ -414,8 +415,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): model, ) return () - effort: Final = "medium" if chat_template_kwargs["enable_thinking"] else "none" - return (("reasoning_effort", effort),) + if chat_template_kwargs["enable_thinking"]: + return () + return (("reasoning_effort", "none"),) @staticmethod def _translate_guided_params(extra_body: Mapping[str, object]) -> tuple[tuple[str, object], ...]: diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index bbb7fb197d0..d25bbd69b91 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1334,7 +1334,7 @@ def test_map_extra_body_params_chat_template_kwargs_enable_thinking(): {"extra_body": {"chat_template_kwargs": {"enable_thinking": True}}}, _REASONING_MODEL, ) - assert enabled == {"reasoning_effort": "medium"} + assert enabled == {} def test_map_extra_body_params_chat_template_kwargs_conflicts_with_reasoning_effort(): @@ -1425,7 +1425,6 @@ def test_map_extra_body_params_multiple_guided_params_rejected(): @pytest.mark.parametrize( "param,value", [ - ("min_tokens", 10), ("stop_token_ids", [1, 2]), ("include_stop_str_in_output", True), ("skip_special_tokens", False), @@ -1440,6 +1439,8 @@ def test_map_extra_body_params_multiple_guided_params_rejected(): ("detokenize", True), ("allowed_token_ids", [1]), ("bad_words", ["foo"]), + ("include_reasoning", False), + ("nvext", {"verbosity": 1}), ], ) def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value, caplog): @@ -1478,9 +1479,10 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): Passing NIM/vLLM extras to litellm.completion must reach the Fireworks request body translated, not verbatim: truncate_prompt_tokens becomes prompt_truncate_len, chat_template_kwargs.enable_thinking becomes - reasoning_effort, min_tokens is dropped, and fireworks-native top_k still - passes through. Asserts on the actual JSON posted to the API, so a revert - of the _complete_fireworks_ai wiring fails this test. + reasoning_effort, include_reasoning is dropped, and min_tokens and + fireworks-native top_k still pass through. Asserts on the actual JSON + posted to the API, so a revert of the _complete_fireworks_ai wiring + fails this test. """ from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -1515,6 +1517,7 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): truncate_prompt_tokens=4096, chat_template_kwargs={"enable_thinking": False}, min_tokens=10, + include_reasoning=False, top_k=40, ) @@ -1523,7 +1526,8 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): assert "truncate_prompt_tokens" not in request_body assert request_body["reasoning_effort"] == "none" assert "chat_template_kwargs" not in request_body - assert "min_tokens" not in request_body + assert "include_reasoning" not in request_body + assert request_body["min_tokens"] == 10 assert request_body["top_k"] == 40 From 431f61b4f7c20b8f722f30c42c279edd19fe6a2d Mon Sep 17 00:00:00 2001 From: Miles Adkins Date: Wed, 5 Aug 2026 16:02:47 -0500 Subject: [PATCH 039/672] fix(fireworks_ai): prefer native values silently on extras conflicts Align with the API gateway translation: instead of raising BadRequestError on alias or competing-constraint conflicts, the explicit Fireworks-native param wins and the NIM/vLLM extra is dropped with a debug log. Covers truncate_prompt_tokens vs prompt_truncate_len, chat_template_kwargs enable_thinking vs reasoning_effort/thinking, guided_* vs response_format (including response_format nested in an explicit extra_body, which the previous conflict check missed), and multiple guided_* params (priority order json, grammar, choice). Malformed non-object chat_template_kwargs is also dropped with a log instead of raising. --- .../llms/fireworks_ai/chat/transformation.py | 108 +++++++---------- .../test_fireworks_ai_chat_transformation.py | 111 +++++++++++------- 2 files changed, 107 insertions(+), 112 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index b5c82129f45..3bacb3cd28e 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -311,7 +311,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): if not isinstance(extra_body, dict): return dict(optional_params) # mutable-ok: JSON request body - self._validate_extra_body_conflicts(extra_body=extra_body, optional_params=optional_params, model=model) stripped: Final = tuple(sorted(k for k in extra_body if k in _NIM_VLLM_STRIP_PARAMS)) if stripped: verbose_logger.debug( @@ -320,9 +319,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): model, ) promoted: Final = ( - *self._translate_truncate_prompt_tokens(extra_body), - *self._translate_chat_template_kwargs(extra_body, model), - *self._translate_guided_params(extra_body), + *self._translate_truncate_prompt_tokens(extra_body, optional_params), + *self._translate_chat_template_kwargs(extra_body, optional_params, model), + *self._translate_guided_params(extra_body, optional_params), ) remaining: Final = tuple((k, v) for k, v in extra_body.items() if k not in _EXTRA_BODY_CONSUMED_PARAMS) base: Final = {k: v for k, v in optional_params.items() if k != "extra_body"} # mutable-ok: JSON request body @@ -332,74 +331,32 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): **({"extra_body": dict(remaining)} if remaining else {}), # mutable-ok: JSON request body } - def _validate_extra_body_conflicts( - self, extra_body: Mapping[str, object], optional_params: Mapping[str, object], model: str - ) -> None: - if "truncate_prompt_tokens" in extra_body and ( - "prompt_truncate_len" in extra_body or "prompt_truncate_len" in optional_params - ): - raise litellm.BadRequestError( - message=( - "Fireworks AI chat completions received both `truncate_prompt_tokens` and " - "`prompt_truncate_len`; they are aliases, send only one." - ), - model=model, - llm_provider="fireworks_ai", - ) - chat_template_kwargs: Final = extra_body.get("chat_template_kwargs") - if ( - isinstance(chat_template_kwargs, dict) - and "enable_thinking" in chat_template_kwargs - and ("reasoning_effort" in optional_params or "thinking" in optional_params) - ): - raise litellm.BadRequestError( - message=( - "Fireworks AI chat completions does not support specifying both " - "`chat_template_kwargs.enable_thinking` and `reasoning_effort`/`thinking` in the same request." - ), - model=model, - llm_provider="fireworks_ai", - ) - guided_params: Final = tuple( - k for k in ("guided_json", "guided_grammar", "guided_choice") if extra_body.get(k) is not None - ) - if len(guided_params) > 1: - raise litellm.BadRequestError( - message=( - f"Fireworks AI chat completions received multiple guided decoding params " - f"{guided_params}; send only one." - ), - model=model, - llm_provider="fireworks_ai", - ) - if guided_params and "response_format" in optional_params: - raise litellm.BadRequestError( - message=( - f"Fireworks AI chat completions received both `{guided_params[0]}` and " - "`response_format`; they are competing output constraints, send only one." - ), - model=model, - llm_provider="fireworks_ai", - ) - @staticmethod - def _translate_truncate_prompt_tokens(extra_body: Mapping[str, object]) -> tuple[tuple[str, object], ...]: + def _translate_truncate_prompt_tokens( + extra_body: Mapping[str, object], optional_params: Mapping[str, object] + ) -> tuple[tuple[str, object], ...]: if extra_body.get("truncate_prompt_tokens") is None: return () + if "prompt_truncate_len" in extra_body or "prompt_truncate_len" in optional_params: + verbose_logger.debug( + "fireworks_ai ignoring truncate_prompt_tokens; explicit prompt_truncate_len takes precedence." + ) + return () return (("prompt_truncate_len", extra_body["truncate_prompt_tokens"]),) def _translate_chat_template_kwargs( - self, extra_body: Mapping[str, object], model: str + self, extra_body: Mapping[str, object], optional_params: Mapping[str, object], model: str ) -> tuple[tuple[str, object], ...]: chat_template_kwargs: Final = extra_body.get("chat_template_kwargs") if chat_template_kwargs is None: return () if not isinstance(chat_template_kwargs, dict): - raise litellm.BadRequestError( - message="Fireworks AI chat completions expects `chat_template_kwargs` to be an object.", - model=model, - llm_provider="fireworks_ai", + verbose_logger.debug( + "fireworks_ai dropping chat_template_kwargs for model=%s; expected an object, got %s.", + model, + type(chat_template_kwargs).__name__, ) + return () other_keys: Final = tuple(sorted(k for k in chat_template_kwargs if k != "enable_thinking")) if other_keys: verbose_logger.debug( @@ -409,6 +366,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): ) if "enable_thinking" not in chat_template_kwargs: return () + if "reasoning_effort" in optional_params or "thinking" in optional_params: + verbose_logger.debug( + "fireworks_ai ignoring chat_template_kwargs.enable_thinking; explicit reasoning_effort/thinking takes precedence." + ) + return () if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): verbose_logger.debug( "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs.enable_thinking.", @@ -420,7 +382,19 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return (("reasoning_effort", "none"),) @staticmethod - def _translate_guided_params(extra_body: Mapping[str, object]) -> tuple[tuple[str, object], ...]: + def _translate_guided_params( + extra_body: Mapping[str, object], optional_params: Mapping[str, object] + ) -> tuple[tuple[str, object], ...]: + has_guided: Final = any( + extra_body.get(key) is not None for key in ("guided_json", "guided_grammar", "guided_choice") + ) + if not has_guided: + return () + if "response_format" in optional_params or "response_format" in extra_body: + verbose_logger.debug( + "fireworks_ai ignoring guided decoding params; explicit response_format takes precedence." + ) + return () if extra_body.get("guided_json") is not None: return (("response_format", _json_schema_response_format(extra_body["guided_json"])),) if extra_body.get("guided_grammar") is not None: @@ -429,13 +403,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): "grammar": extra_body["guided_grammar"], } return (("response_format", grammar_response_format),) - if extra_body.get("guided_choice") is not None: - choice_schema: Final = { # mutable-ok: JSON request body - "type": "string", - "enum": extra_body["guided_choice"], - } - return (("response_format", _json_schema_response_format(choice_schema)),) - return () + choice_schema: Final = { # mutable-ok: JSON request body + "type": "string", + "enum": extra_body["guided_choice"], + } + return (("response_format", _json_schema_response_format(choice_schema)),) def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]: for tool in tools: diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index d25bbd69b91..48d868b5846 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1308,18 +1308,19 @@ def test_map_extra_body_params_translates_truncate_prompt_tokens(): assert result == {"prompt_truncate_len": 4096} -def test_map_extra_body_params_truncate_prompt_tokens_conflicts_with_alias(): +def test_map_extra_body_params_truncate_prompt_tokens_native_wins(): config = FireworksAIConfig() - with pytest.raises(litellm.BadRequestError, match="aliases"): - config.map_extra_body_params( - {"prompt_truncate_len": 2048, "extra_body": {"truncate_prompt_tokens": 4096}}, - _REASONING_MODEL, - ) - with pytest.raises(litellm.BadRequestError, match="aliases"): - config.map_extra_body_params( - {"extra_body": {"truncate_prompt_tokens": 4096, "prompt_truncate_len": 2048}}, - _REASONING_MODEL, - ) + top_level = config.map_extra_body_params( + {"prompt_truncate_len": 2048, "extra_body": {"truncate_prompt_tokens": 4096}}, + _REASONING_MODEL, + ) + assert top_level == {"prompt_truncate_len": 2048} + + nested = config.map_extra_body_params( + {"extra_body": {"truncate_prompt_tokens": 4096, "prompt_truncate_len": 2048}}, + _REASONING_MODEL, + ) + assert nested == {"extra_body": {"prompt_truncate_len": 2048}} def test_map_extra_body_params_chat_template_kwargs_enable_thinking(): @@ -1337,28 +1338,29 @@ def test_map_extra_body_params_chat_template_kwargs_enable_thinking(): assert enabled == {} -def test_map_extra_body_params_chat_template_kwargs_conflicts_with_reasoning_effort(): +def test_map_extra_body_params_chat_template_kwargs_native_reasoning_effort_wins(): config = FireworksAIConfig() - with pytest.raises(litellm.BadRequestError, match="enable_thinking"): - config.map_extra_body_params( - { - "reasoning_effort": "high", - "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, - }, - _REASONING_MODEL, - ) + result = config.map_extra_body_params( + { + "reasoning_effort": "high", + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + }, + _REASONING_MODEL, + ) + assert result == {"reasoning_effort": "high"} -def test_map_extra_body_params_chat_template_kwargs_conflicts_with_thinking(): +def test_map_extra_body_params_chat_template_kwargs_native_thinking_wins(): config = FireworksAIConfig() - with pytest.raises(litellm.BadRequestError, match="enable_thinking"): - config.map_extra_body_params( - { - "thinking": {"type": "enabled", "budget_tokens": 4096}, - "extra_body": {"chat_template_kwargs": {"enable_thinking": True}}, - }, - _REASONING_MODEL, - ) + thinking = {"type": "enabled", "budget_tokens": 4096} + result = config.map_extra_body_params( + { + "thinking": thinking, + "extra_body": {"chat_template_kwargs": {"enable_thinking": True}}, + }, + _REASONING_MODEL, + ) + assert result == {"thinking": thinking} def test_map_extra_body_params_chat_template_kwargs_dropped_for_non_reasoning_model(): @@ -1370,6 +1372,15 @@ def test_map_extra_body_params_chat_template_kwargs_dropped_for_non_reasoning_mo assert result == {} +def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": "enable_thinking"}}, + _REASONING_MODEL, + ) + assert result == {} + + def test_map_extra_body_params_guided_json(): config = FireworksAIConfig() schema = {"type": "object", "properties": {"x": {"type": "string"}}} @@ -1401,25 +1412,37 @@ def test_map_extra_body_params_guided_grammar_and_choice(): } -def test_map_extra_body_params_guided_conflicts_with_response_format(): +def test_map_extra_body_params_guided_native_response_format_wins(): config = FireworksAIConfig() - with pytest.raises(litellm.BadRequestError, match="response_format"): - config.map_extra_body_params( - { - "response_format": {"type": "json_object"}, - "extra_body": {"guided_json": {"type": "object"}}, - }, - _REASONING_MODEL, - ) + top_level = config.map_extra_body_params( + { + "response_format": {"type": "json_object"}, + "extra_body": {"guided_json": {"type": "object"}}, + }, + _REASONING_MODEL, + ) + assert top_level == {"response_format": {"type": "json_object"}} + + nested_format = {"type": "json_object"} + nested = config.map_extra_body_params( + {"extra_body": {"guided_json": {"type": "object"}, "response_format": nested_format}}, + _REASONING_MODEL, + ) + assert nested == {"extra_body": {"response_format": nested_format}} -def test_map_extra_body_params_multiple_guided_params_rejected(): +def test_map_extra_body_params_multiple_guided_params_priority_order(): config = FireworksAIConfig() - with pytest.raises(litellm.BadRequestError, match="multiple guided decoding params"): - config.map_extra_body_params( - {"extra_body": {"guided_json": {"type": "object"}, "guided_grammar": "root ::= 'x'"}}, - _REASONING_MODEL, - ) + result = config.map_extra_body_params( + {"extra_body": {"guided_grammar": "root ::= 'x'", "guided_json": {"type": "object"}}}, + _REASONING_MODEL, + ) + assert result == { + "response_format": { + "type": "json_schema", + "json_schema": {"schema": {"type": "object"}}, + } + } @pytest.mark.parametrize( From 2ba4e917666e84ca9b83d37ecbe807226e149020 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:38:08 -0700 Subject: [PATCH 040/672] feat(guardrails): add scan_only_tool_results to scope unified guardrails to tool results --- .../chat/guardrail_translation/handler.py | 35 ++++-- .../base_llm/guardrail_translation/utils.py | 53 +++++++- .../chat/guardrail_translation/handler.py | 33 +++-- .../proxy/guardrails/guardrail_registry.py | 12 +- litellm/types/guardrails.py | 10 ++ .../test_anthropic_guardrail_handler.py | 113 ++++++++++++++++++ .../test_openai_guardrail_handler.py | 68 +++++++++++ 7 files changed, 288 insertions(+), 36 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 3662389900b..535f4b7ae61 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,10 +26,10 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im ) from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, - openai_messages_without_system, - openai_messages_without_tool, + filtered_structured_messages, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -326,19 +326,25 @@ class AnthropicMessagesHandler(BaseTranslation): skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply) skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply) + scan_only_tool_results: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply) chat_completion_compatible_request: Final = self._translate_to_openai(data) - structured_messages = cast( - list[AllMessageValues], - chat_completion_compatible_request.get("messages", []), + structured_messages: Final = list( + filtered_structured_messages( + cast( + list[AllMessageValues], + chat_completion_compatible_request.get("messages", []), + ), + scan_only_tool_results=scan_only_tool_results, + skip_system=skip_system, + skip_tool=skip_tool, + ) ) - if skip_system: - structured_messages = openai_messages_without_system(structured_messages) - if skip_tool: - structured_messages = openai_messages_without_tool(structured_messages) - tools_to_check: Final[list[ChatCompletionToolParam]] = chat_completion_compatible_request.get("tools", []) + tools_to_check: Final[list[ChatCompletionToolParam]] = ( + [] if scan_only_tool_results else chat_completion_compatible_request.get("tools", []) + ) # Step 1: Extract all text content and images extracted: Final = tuple( @@ -347,6 +353,7 @@ class AnthropicMessagesHandler(BaseTranslation): msg_idx=msg_idx, skip_system_message=skip_system, skip_tool_message=skip_tool, + scan_only_tool_results=scan_only_tool_results, ) for msg_idx, message in enumerate(messages) ) @@ -461,6 +468,7 @@ class AnthropicMessagesHandler(BaseTranslation): msg_idx: int, skip_system_message: bool = False, skip_tool_message: bool = False, + scan_only_tool_results: bool = False, ) -> ExtractedInput: """ Extract text content and images from a message. @@ -471,6 +479,8 @@ class AnthropicMessagesHandler(BaseTranslation): content: Final = message.get("content", None) if isinstance(content, str): + if scan_only_tool_results: + return EMPTY_EXTRACTED_INPUT return ExtractedInput(scanned=(ScannedText(content, MessageContentTarget(msg_idx)),), images=()) if not isinstance(content, list): return EMPTY_EXTRACTED_INPUT @@ -481,6 +491,7 @@ class AnthropicMessagesHandler(BaseTranslation): msg_idx=msg_idx, content_idx=content_idx, skip_tool_message=skip_tool_message, + scan_only_tool_results=scan_only_tool_results, ) for content_idx, content_item in enumerate(content) if isinstance(content_item, dict) @@ -497,12 +508,16 @@ class AnthropicMessagesHandler(BaseTranslation): msg_idx: int, content_idx: int, skip_tool_message: bool, + scan_only_tool_results: bool = False, ) -> ExtractedInput: if content_item.get("type") == "tool_result": if skip_tool_message: return EMPTY_EXTRACTED_INPUT return cls._extract_tool_result(content_item=content_item, msg_idx=msg_idx, content_idx=content_idx) + if scan_only_tool_results: + return EMPTY_EXTRACTED_INPUT + text_str: Final = content_item.get("text", None) return ExtractedInput( scanned=( diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 17cc0f118d6..e365913f2e1 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from collections.abc import Sequence from typing import Any, Final from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage @@ -113,13 +114,53 @@ def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool: return bool(getattr(litellm, "skip_tool_message_in_guardrail", False)) +def _message_role(message: AllMessageValues) -> str: + return str((message or {}).get("role") or "").lower() + + def openai_messages_without_system( - messages: list[AllMessageValues], -) -> list[AllMessageValues]: - return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"] + messages: Sequence[AllMessageValues], +) -> tuple[AllMessageValues, ...]: + return tuple(m for m in messages if _message_role(m) != "system") def openai_messages_without_tool( - messages: list[AllMessageValues], -) -> list[AllMessageValues]: - return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"] + messages: Sequence[AllMessageValues], +) -> tuple[AllMessageValues, ...]: + return tuple(m for m in messages if _message_role(m) != "tool") + + +def openai_messages_only_tool( + messages: Sequence[AllMessageValues], +) -> tuple[AllMessageValues, ...]: + return tuple(m for m in messages if _message_role(m) == "tool") + + +def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: Any) -> bool: + return getattr(guardrail_to_apply, "scan_only_tool_results", None) is True + + +def role_out_of_guardrail_scope( + role: str, + *, + skip_system_message: bool, + skip_tool_message: bool, + scan_only_tool_results: bool = False, +) -> bool: + if skip_system_message and role == "system": + return True + if skip_tool_message and role == "tool": + return True + return scan_only_tool_results and role != "tool" + + +def filtered_structured_messages( + messages: Sequence[AllMessageValues], + *, + scan_only_tool_results: bool, + skip_system: bool, + skip_tool: bool, +) -> tuple[AllMessageValues, ...]: + scoped: Final = openai_messages_only_tool(messages) if scan_only_tool_results else tuple(messages) + without_system: Final = openai_messages_without_system(scoped) if skip_system else scoped + return openai_messages_without_tool(without_system) if skip_tool else without_system diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3988326f2c2..67550890d2d 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -23,10 +23,11 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import ( StreamTransformSink, ) from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, - openai_messages_without_system, - openai_messages_without_tool, + filtered_structured_messages, + role_out_of_guardrail_scope, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -82,6 +83,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply) skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply) + scan_only_tool_results: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply) texts_to_check: Final[list[str]] = [] images_to_check: Final[list[str]] = [] @@ -101,6 +103,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_call_task_mappings=tool_call_task_mappings, skip_system_message=skip_system, skip_tool_message=skip_tool, + scan_only_tool_results=scan_only_tool_results, ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -110,13 +113,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs["images"] = images_to_check if tool_calls_to_check: inputs["tool_calls"] = tool_calls_to_check - structured_messages = self.get_structured_messages(data) + structured_messages: Final = self.get_structured_messages(data) if structured_messages: - if skip_system: - structured_messages = openai_messages_without_system(structured_messages) - if skip_tool: - structured_messages = openai_messages_without_tool(structured_messages) - inputs["structured_messages"] = structured_messages + inputs["structured_messages"] = list( + filtered_structured_messages( + structured_messages, + scan_only_tool_results=scan_only_tool_results, + skip_system=skip_system, + skip_tool=skip_tool, + ) + ) # Pass tools (function definitions) to the guardrail tools: Final = data.get("tools") if tools: @@ -194,16 +200,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_call_task_mappings: list[tuple[int, int]], skip_system_message: bool = False, skip_tool_message: bool = False, + scan_only_tool_results: bool = False, ) -> None: """ Extract text content, images, and tool calls from a message. Override this method to customize text/image/tool call extraction logic. """ - role: Final = str(message.get("role") or "").lower() - if skip_system_message and role == "system": - return - if skip_tool_message and role == "tool": + if role_out_of_guardrail_scope( + str(message.get("role") or "").lower(), + skip_system_message=skip_system_message, + skip_tool_message=skip_tool_message, + scan_only_tool_results=scan_only_tool_results, + ): return content: Final = message.get("content", None) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index f77588cf087..e9e61283c1a 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -487,16 +487,12 @@ class InMemoryGuardrailHandler: raise ValueError(f"Unsupported guardrail: {guardrail_type}") if custom_guardrail_callback is not None: - setattr( - custom_guardrail_callback, + for scoping_param in ( "skip_system_message_in_guardrail", - getattr(litellm_params, "skip_system_message_in_guardrail", None), - ) - setattr( - custom_guardrail_callback, "skip_tool_message_in_guardrail", - getattr(litellm_params, "skip_tool_message_in_guardrail", None), - ) + "scan_only_tool_results", + ): + setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) configured_run_in_parallel: Final = getattr(litellm_params, "run_in_parallel", None) if configured_run_in_parallel is not None: custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index e7ad5cb801d..3eb8faf91dc 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -757,6 +757,16 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) + scan_only_tool_results: Optional[bool] = Field( + default=None, + description=( + "When True, unified guardrails only evaluate tool results, the untrusted data an " + "agent feeds back into the model, and skip system, user, and assistant content. " + "Intended for agent harnesses whose own prompt scaffolding is trusted but often " + "trips prompt-attack detectors." + ), + ) + # Lakera specific params category_thresholds: Optional[LakeraCategoryThresholds] = Field( default=None, diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index dff3390af12..e90ae579d6d 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -760,3 +760,116 @@ class TestAnthropicMessagesToolResultScanning: assert "skip me POISON" not in guardrail.seen_texts assert messages[1]["content"][0]["content"] == "skip me POISON" assert messages[0]["content"] == "keep me [BLOCKED]" + + +class InputsRecordingGuardrail(MockMaskingGuardrail): + def __init__(self): + super().__init__(guardrail_name="scan-only-capture") + self.captured_inputs: Optional[GenericGuardrailAPIInputs] = None + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + self.captured_inputs = inputs + return await super().apply_guardrail(inputs, request_data, input_type, logging_obj) + + +class TestAnthropicMessagesScanOnlyToolResults: + def _guardrail(self): + guardrail = InputsRecordingGuardrail() + guardrail.scan_only_tool_results = True + return guardrail + + @pytest.mark.asyncio + async def test_scan_narrows_to_tool_results_and_write_back_stays_aligned(self): + handler = AnthropicMessagesHandler() + guardrail = self._guardrail() + data = { + "model": "claude-sonnet-4-5", + "system": "You are a trusted agent harness with POISON heuristics.", + "tools": [ + { + "name": "Bash", + "description": "run a command", + "input_schema": {"type": "object", "properties": {}}, + } + ], + "messages": [ + {"role": "user", "content": "scaffolding POISON prompt"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu1", "name": "Bash", "input": {"cmd": "curl"}}], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "sibling POISON text"}, + {"type": "tool_result", "tool_use_id": "tu1", "content": "fetched POISON page"}, + ], + }, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.seen_texts == ["fetched POISON page"], ( + "only the tool_result payload may reach the guardrail" + ) + assert guardrail.captured_inputs is not None + assert guardrail.captured_inputs.get("tools") is None + assert [m["role"] for m in guardrail.captured_inputs["structured_messages"]] == ["tool"] + assert data["messages"][2]["content"][1]["content"] == "fetched [BLOCKED] page" + assert data["messages"][0]["content"] == "scaffolding POISON prompt", ( + "out-of-scope content must come back untouched, not masked or dropped" + ) + assert data["messages"][2]["content"][0]["text"] == "sibling POISON text" + + @pytest.mark.asyncio + async def test_guardrail_is_not_called_when_the_request_has_no_tool_results(self): + handler = AnthropicMessagesHandler() + guardrail = self._guardrail() + data = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "What is 2 plus 2?"}], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.captured_inputs is None + assert guardrail.seen_texts == [] + + @pytest.mark.asyncio + async def test_images_are_scoped_the_same_way_as_texts(self): + handler = AnthropicMessagesHandler() + guardrail = self._guardrail() + data = { + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "user", + "content": [{"type": "image", "source": {"type": "base64", "data": "USER_IMG"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu1", + "content": [ + {"type": "text", "text": "screenshot POISON"}, + {"type": "image", "source": {"type": "base64", "data": "TOOL_IMG"}}, + ], + } + ], + }, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.captured_inputs is not None + assert guardrail.captured_inputs.get("images") == ["TOOL_IMG"] diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 7730b664c5e..c8a1b98aa82 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1229,3 +1229,71 @@ class TestIncrementalScanRespectsSkipFlags: assert mock_api.call_count == 1 scanned = [m["content"] for m in mock_api.call_args.kwargs["messages"]] assert scanned == ["It is sunny in Paris.", "And tomorrow?"] + + +class TestScanOnlyToolResults: + def _bedrock_guardrail(self): + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail + + guardrail = BedrockGuardrail( + guardrail_name="bedrock-scan-only-tool-results", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + default_on=True, + ) + guardrail.scan_only_tool_results = True + return guardrail + + @pytest.mark.asyncio + async def test_only_tool_role_content_is_scanned(self): + from unittest.mock import AsyncMock, patch + + handler = OpenAIChatCompletionsHandler() + guardrail = self._bedrock_guardrail() + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT-not-scanned"}, + {"role": "user", "content": "USER-PROMPT-not-scanned"}, + { + "role": "assistant", + "content": "ASSISTANT-not-scanned", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "report.html"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT-scanned"}, + ] + } + with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api: + mock_api.return_value = {"action": "NONE", "output": [], "outputs": []} + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + assert mock_api.call_count == 1 + scanned = [m["content"] for m in mock_api.call_args.kwargs["messages"]] + assert scanned == ["TOOL-RESULT-scanned"] + + @pytest.mark.parametrize("flag_value", [None, "false", 0, object()]) + @pytest.mark.asyncio + async def test_scope_narrows_only_when_the_flag_is_actually_true(self, flag_value): + from unittest.mock import AsyncMock, patch + + handler = OpenAIChatCompletionsHandler() + guardrail = self._bedrock_guardrail() + guardrail.scan_only_tool_results = flag_value + data = { + "messages": [ + {"role": "user", "content": "USER-PROMPT"}, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}, + ] + } + with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api: + mock_api.return_value = {"action": "NONE", "output": [], "outputs": []} + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + assert mock_api.call_count == 1 + scanned = [m["content"] for m in mock_api.call_args.kwargs["messages"]] + assert scanned == ["USER-PROMPT", "TOOL-RESULT"], ( + "anything but an explicit True must leave the whole request in scope" + ) From 3d673f9534f961c7f709b0a70063f349ab7cfd2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:58:20 +0000 Subject: [PATCH 041/672] fix(managed_files): skip unparseable rows when listing managed files get_user_created_file_ids validated every row's file_object without a guard, so a single row failing OpenAIFileObject validation raised ValidationError and turned the whole GET /v1/files response into a 500. #35365 covered the null case only, leaving malformed or partial rows able to take the entire listing down. Rows now parse through a helper that returns None on failure and logs a warning, matching how list_user_batches already tolerates rows it cannot parse, so one bad row costs its own entry instead of the caller's whole listing. Null rows stay silent since the batch cost poller registers those legitimately. Refs #35361 --- .../proxy/hooks/managed_files.py | 23 +++++++++++++++++-- .../proxy/test_managed_files_hook.py | 23 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index ec47b6ac0e6..2349b618a28 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -73,6 +73,20 @@ else: PrismaClient = Any +def _parse_managed_file_object( + raw_file_object: object, unified_file_id: str +) -> Optional[OpenAIFileObject]: + if raw_file_object is None: + return None + try: + return OpenAIFileObject.model_validate(raw_file_object) + except Exception as e: + verbose_logger.warning( + f"Failed to parse managed file object {unified_file_id}: {e}" + ) + return None + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -383,9 +397,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): } ) return [ - OpenAIFileObject.model_validate(file_object.file_object) + parsed_file_object for file_object in file_ids - if file_object.file_object is not None + if ( + parsed_file_object := _parse_managed_file_object( + file_object.file_object, file_object.unified_file_id + ) + ) + is not None ] async def check_managed_file_id_access( diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py index 4a4aa7aa5ea..4da6de6353f 100644 --- a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py +++ b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py @@ -154,6 +154,29 @@ async def test_get_user_created_file_ids_skips_rows_without_file_object(): assert [file.id for file in files] == ["file-output-abc"] +@pytest.mark.asyncio +async def test_get_user_created_file_ids_skips_unparseable_rows(): + managed_files = _make_managed_files_instance() + managed_files.prisma_client.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[ + MagicMock( + file_object={"id": "file-corrupt", "object": "file"}, + unified_file_id="unified-corrupt", + ), + MagicMock( + file_object=_make_file_object().model_dump(), + unified_file_id="unified-valid", + ), + ] + ) + + files = await managed_files.get_user_created_file_ids( + _make_user_api_key_dict(), ["file-output-abc"] + ) + + assert [file.id for file in files] == ["file-output-abc"] + + @pytest.mark.asyncio async def test_should_fallback_when_no_router(): """ From 1b6f3cebf1a4a4804a9bd9a0c3287cfc0d07c971 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 00:16:05 +0000 Subject: [PATCH 042/672] fix(managed_files): log sanitized validation errors when skipping rows The skip warning interpolated the full pydantic ValidationError, whose string embeds input_value with the rejected row's contents. Managed-file rows carry a caller-supplied filename, so a malformed row copied that into operational logs. Log the error locations, types, and messages via errors() with input, url, and context excluded, keeping the field-level diagnostics without the values. Non-validation failures fall back to the exception type. --- .../proxy/hooks/managed_files.py | 9 ++++++++- .../proxy/test_managed_files_hook.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 2349b618a28..688ffb35ff7 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast from fastapi import HTTPException +from pydantic import ValidationError import litellm from litellm import Router, verbose_logger @@ -80,9 +81,15 @@ def _parse_managed_file_object( return None try: return OpenAIFileObject.model_validate(raw_file_object) + except ValidationError as e: + verbose_logger.warning( + f"Failed to parse managed file object {unified_file_id}: " + f"{e.errors(include_input=False, include_url=False, include_context=False)}" + ) + return None except Exception as e: verbose_logger.warning( - f"Failed to parse managed file object {unified_file_id}: {e}" + f"Failed to parse managed file object {unified_file_id}: {type(e).__name__}" ) return None diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py index 4da6de6353f..6397e0be247 100644 --- a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py +++ b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py @@ -6,6 +6,7 @@ async_post_call_success_hook when processing completed batch responses. """ import json +import logging import pytest from typing import Optional @@ -154,6 +155,24 @@ async def test_get_user_created_file_ids_skips_rows_without_file_object(): assert [file.id for file in files] == ["file-output-abc"] +@pytest.mark.asyncio +async def test_parse_managed_file_object_warning_omits_rejected_values(caplog): + from litellm_enterprise.proxy.hooks.managed_files import ( + _parse_managed_file_object, + ) + + with caplog.at_level(logging.WARNING): + parsed = _parse_managed_file_object( + {"id": "file-corrupt", "object": "file", "filename": "confidential.jsonl"}, + "unified-corrupt", + ) + + assert parsed is None + assert "unified-corrupt" in caplog.text + assert "bytes" in caplog.text + assert "confidential.jsonl" not in caplog.text + + @pytest.mark.asyncio async def test_get_user_created_file_ids_skips_unparseable_rows(): managed_files = _make_managed_files_instance() From d70e10982a46f728c6d5a431fd8692a85b3ebf23 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:21:58 -0700 Subject: [PATCH 043/672] fix(guardrails): keep tool-results-only scans off function definitions and merge scoped write-backs Gate the OpenAI handler's tools forwarding behind scan_only_tool_results, matching the Anthropic handler, so a tool-results-only scan can no longer evaluate or rewrite trusted function definitions. When a guardrail returns a replacement structured_messages list, substitute the returned messages back into the positions their scoped originals came from instead of installing the scoped list as the whole conversation, so out-of-scope messages (system prompt, prior turns) survive redaction on both the OpenAI and Anthropic paths. --- .../chat/guardrail_translation/handler.py | 32 +++++--- .../base_llm/guardrail_translation/utils.py | 58 ++++++++++--- .../chat/guardrail_translation/handler.py | 26 +++--- .../test_anthropic_guardrail_handler.py | 54 +++++++++++++ .../test_openai_guardrail_handler.py | 81 +++++++++++++++++++ 5 files changed, 216 insertions(+), 35 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 535f4b7ae61..c25fa624f7f 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -29,7 +29,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, - filtered_structured_messages, + merge_guardrailed_scoped_messages, + scoped_structured_message_indices, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -330,17 +331,17 @@ class AnthropicMessagesHandler(BaseTranslation): chat_completion_compatible_request: Final = self._translate_to_openai(data) - structured_messages: Final = list( - filtered_structured_messages( - cast( - list[AllMessageValues], - chat_completion_compatible_request.get("messages", []), - ), - scan_only_tool_results=scan_only_tool_results, - skip_system=skip_system, - skip_tool=skip_tool, - ) + full_structured_messages: Final = cast( + list[AllMessageValues], + chat_completion_compatible_request.get("messages", []), ) + scoped_message_indices: Final = scoped_structured_message_indices( + full_structured_messages, + scan_only_tool_results=scan_only_tool_results, + skip_system=skip_system, + skip_tool=skip_tool, + ) + structured_messages: Final = [full_structured_messages[index] for index in scoped_message_indices] tools_to_check: Final[list[ChatCompletionToolParam]] = ( [] if scan_only_tool_results else chat_completion_compatible_request.get("tools", []) @@ -402,7 +403,14 @@ class AnthropicMessagesHandler(BaseTranslation): guardrailed_structured_messages is not None and guardrailed_structured_messages is not original_structured_messages ): - self._write_back_structured_messages(data, guardrailed_structured_messages) + self._write_back_structured_messages( + data, + merge_guardrailed_scoped_messages( + full_messages=full_structured_messages, + scoped_indices=scoped_message_indices, + guardrailed_scoped=guardrailed_structured_messages, + ), + ) else: # Step 3: Map guardrail responses back to original message structure await self._apply_guardrail_responses_to_input( diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index e365913f2e1..fcd504fee2f 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from typing import Any, Final from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage @@ -130,12 +130,6 @@ def openai_messages_without_tool( return tuple(m for m in messages if _message_role(m) != "tool") -def openai_messages_only_tool( - messages: Sequence[AllMessageValues], -) -> tuple[AllMessageValues, ...]: - return tuple(m for m in messages if _message_role(m) == "tool") - - def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: Any) -> bool: return getattr(guardrail_to_apply, "scan_only_tool_results", None) is True @@ -154,13 +148,53 @@ def role_out_of_guardrail_scope( return scan_only_tool_results and role != "tool" -def filtered_structured_messages( +def scoped_structured_message_indices( messages: Sequence[AllMessageValues], *, scan_only_tool_results: bool, skip_system: bool, skip_tool: bool, -) -> tuple[AllMessageValues, ...]: - scoped: Final = openai_messages_only_tool(messages) if scan_only_tool_results else tuple(messages) - without_system: Final = openai_messages_without_system(scoped) if skip_system else scoped - return openai_messages_without_tool(without_system) if skip_tool else without_system +) -> tuple[int, ...]: + return tuple( + index + for index, message in enumerate(messages) + if not role_out_of_guardrail_scope( + _message_role(message), + skip_system_message=skip_system, + skip_tool_message=skip_tool, + scan_only_tool_results=scan_only_tool_results, + ) + ) + + +def merge_guardrailed_scoped_messages( + full_messages: Sequence[AllMessageValues], + scoped_indices: Sequence[int], + guardrailed_scoped: Sequence[AllMessageValues], +) -> list[AllMessageValues]: + """Substitute guardrail-returned messages back into the full conversation. + + Guardrails only ever see the scoped subset of messages, so a replacement + list they hand back describes that subset, not the whole request. Writing + it over ``data["messages"]`` wholesale would silently drop every + out-of-scope message (system prompt, prior turns). Instead, swap each + returned message into the position its scoped original came from; extra + returned messages land after the last scoped position, and scoped + originals without a counterpart are treated as removed by the guardrail. + When nothing was filtered out this degenerates to the returned list + itself, preserving wholesale-replacement behavior for unscoped guardrails. + """ + replacements: Final = dict(zip(scoped_indices, guardrailed_scoped)) + removed: Final = frozenset(scoped_indices[len(guardrailed_scoped) :]) + appended: Final = tuple(guardrailed_scoped[len(scoped_indices) :]) + last_scoped_index: Final = scoped_indices[-1] if scoped_indices else None + + def _merged() -> Iterator[AllMessageValues]: + for index, message in enumerate(full_messages): + if index in removed: + continue + yield replacements.get(index, message) + if index == last_scoped_index: + yield from appended + + return list(_merged()) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 67550890d2d..9d7fe6ce2a8 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -26,8 +26,9 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, - filtered_structured_messages, + merge_guardrailed_scoped_messages, role_out_of_guardrail_scope, + scoped_structured_message_indices, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -114,18 +115,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if tool_calls_to_check: inputs["tool_calls"] = tool_calls_to_check structured_messages: Final = self.get_structured_messages(data) + scoped_message_indices: Final = scoped_structured_message_indices( + structured_messages or [], + scan_only_tool_results=scan_only_tool_results, + skip_system=skip_system, + skip_tool=skip_tool, + ) if structured_messages: - inputs["structured_messages"] = list( - filtered_structured_messages( - structured_messages, - scan_only_tool_results=scan_only_tool_results, - skip_system=skip_system, - skip_tool=skip_tool, - ) - ) + inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices] # Pass tools (function definitions) to the guardrail tools: Final = data.get("tools") - if tools: + if tools and not scan_only_tool_results: inputs["tools"] = tools # Include model information if available model: Final = data.get("model") @@ -151,7 +151,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrailed_structured_messages is not None and guardrailed_structured_messages is not original_structured_messages ): - data["messages"] = guardrailed_structured_messages + data["messages"] = merge_guardrailed_scoped_messages( + full_messages=structured_messages or [], + scoped_indices=scoped_message_indices, + guardrailed_scoped=guardrailed_structured_messages, + ) else: # Step 3: Map guardrail responses back to original message structure if guardrailed_texts and texts_to_check: diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index e90ae579d6d..a016e1a2deb 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -5,6 +5,7 @@ Tests the handler's ability to process streaming output for Anthropic Messages A with guardrail transformations, specifically testing edge cases with empty choices. """ +import json import os import sys from typing import Any, Literal, Optional @@ -778,12 +779,65 @@ class InputsRecordingGuardrail(MockMaskingGuardrail): return await super().apply_guardrail(inputs, request_data, input_type, logging_obj) +class StructuredMessagesRewritingGuardrail(CustomGuardrail): + """Returns a new structured_messages list with a canary redacted, like redaction guardrails do.""" + + def __init__(self): + super().__init__(guardrail_name="structured-rewrite") + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + structured = inputs.get("structured_messages") or [] + inputs["structured_messages"] = [ + json.loads(json.dumps(message).replace("POISON", "[BLOCKED]")) for message in structured + ] + return inputs + + class TestAnthropicMessagesScanOnlyToolResults: def _guardrail(self): guardrail = InputsRecordingGuardrail() guardrail.scan_only_tool_results = True return guardrail + @pytest.mark.asyncio + async def test_structured_write_back_merges_into_the_full_conversation(self): + handler = AnthropicMessagesHandler() + guardrail = StructuredMessagesRewritingGuardrail() + guardrail.scan_only_tool_results = True + data = { + "model": "claude-sonnet-4-5", + "system": "You are a careful agent harness.", + "messages": [ + {"role": "user", "content": "fetch the page"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu1", "name": "Bash", "input": {"cmd": "curl"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "tu1", "content": "fetched POISON page"}], + }, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert data["system"] == "You are a careful agent harness." + assert [m["role"] for m in data["messages"]] == ["user", "assistant", "user"], ( + "a redacting guardrail must not strip out-of-scope turns from the request" + ) + serialized = json.dumps(data["messages"]) + assert "fetch the page" in serialized + assert "tool_use" in serialized + assert "fetched [BLOCKED] page" in serialized + assert "POISON" not in serialized + @pytest.mark.asyncio async def test_scan_narrows_to_tool_results_and_write_back_stays_aligned(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c8a1b98aa82..907da66e5bf 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1231,6 +1231,28 @@ class TestIncrementalScanRespectsSkipFlags: assert scanned == ["It is sunny in Paris.", "And tomorrow?"] +class StructuredRedactionGuardrail(CustomGuardrail): + """Captures inputs and returns a new structured_messages list with a canary redacted.""" + + def __init__(self): + super().__init__(guardrail_name="structured-redaction") + self.captured_inputs: Optional[GenericGuardrailAPIInputs] = None + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + self.captured_inputs = inputs + structured = inputs.get("structured_messages") or [] + inputs["structured_messages"] = [ + {**m, "content": str(m.get("content", "")).replace("POISON", "[BLOCKED]")} for m in structured + ] + return inputs + + class TestScanOnlyToolResults: def _bedrock_guardrail(self): from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail @@ -1297,3 +1319,62 @@ class TestScanOnlyToolResults: assert scanned == ["USER-PROMPT", "TOOL-RESULT"], ( "anything but an explicit True must leave the whole request in scope" ) + + @pytest.mark.parametrize("scan_only_tool_results", [True, False]) + @pytest.mark.asyncio + async def test_function_definitions_are_scoped_out_with_the_tool_results_flag(self, scan_only_tool_results): + handler = OpenAIChatCompletionsHandler() + guardrail = StructuredRedactionGuardrail() + guardrail.scan_only_tool_results = scan_only_tool_results + tools = [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object", "properties": {}}}, + } + ] + data = { + "messages": [ + {"role": "user", "content": "read the report"}, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}, + ], + "tools": tools, + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.captured_inputs is not None + expected_tools = None if scan_only_tool_results else tools + assert guardrail.captured_inputs.get("tools") == expected_tools, ( + "function definitions must stay out of a tool-results-only scan" + ) + + @pytest.mark.asyncio + async def test_structured_write_back_keeps_out_of_scope_messages(self): + handler = OpenAIChatCompletionsHandler() + guardrail = StructuredRedactionGuardrail() + guardrail.scan_only_tool_results = True + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT"}, + {"role": "user", "content": "fetch the page"}, + { + "role": "assistant", + "content": "fetching", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "fetch", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "page says POISON here"}, + {"role": "user", "content": "and then?"}, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [m["role"] for m in data["messages"]] == ["system", "user", "assistant", "tool", "user"], ( + "a redacting guardrail must not strip out-of-scope messages from the request" + ) + assert data["messages"][0]["content"] == "SYSTEM-PROMPT" + assert data["messages"][3]["content"] == "page says [BLOCKED] here" + assert data["messages"][3]["tool_call_id"] == "call_1" + assert data["messages"][4]["content"] == "and then?" From eef908d4ad542e8003f22d76dd12ad559f25733d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:26:21 -0700 Subject: [PATCH 044/672] fix(batches): register managed output files on batch cancel update_batch_in_database now fetches the batch row by unified_object_id when the caller omits db_batch_object, so the cancel endpoint attributes newly registered output and error files to the batch owner and returns unified managed ids instead of raw provider ids. Idempotent cancels that do not change the stored status also skip the redundant DB write now. Repair two pre-existing mock tests in test_openai_batches_endpoint.py that asserted values inside lazy percent-format log strings, and give the cancel test's prisma mock an awaitable find_first. --- .../openai_files_endpoints/common_utils.py | 17 +++-- .../test_openai_batches_endpoint.py | 7 +- ..._batch_update_db_managed_output_file_id.py | 64 ++++++++++++++++++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 37b51e1d3af..290045a5a87 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1141,7 +1141,7 @@ async def update_batch_in_database( managed_files_obj: The managed_files proxy hook object prisma_client: Prisma database client verbose_proxy_logger: Logger instance - db_batch_object: Optional existing database object (for comparison) + db_batch_object: Optional existing database object; fetched by unified_object_id when omitted operation: Description of operation ("update", "cancel", etc.) user_api_key_dict: Optional auth context for creating managed file IDs """ @@ -1154,6 +1154,12 @@ async def update_batch_in_database( if not prisma_client: return + effective_db_batch_object: Final = ( + db_batch_object + if db_batch_object is not None + else await ManagedObjectRepository(prisma_client).table.find_first(where={"unified_object_id": batch_id}) + ) + # Always normalize the response's file IDs to unified managed IDs # (mutates in place) so the caller returns unified IDs to the user # even when we skip the DB update below for an unchanged status. @@ -1163,16 +1169,17 @@ async def update_batch_in_database( prisma_client=prisma_client, verbose_proxy_logger=verbose_proxy_logger, user_api_key_dict=user_api_key_dict, - db_batch_object=db_batch_object, + db_batch_object=effective_db_batch_object, + unified_batch_id=unified_batch_id, ) # Only update if status has changed (when db_batch_object is provided) - if db_batch_object and response.status == db_batch_object.status: + if effective_db_batch_object and response.status == effective_db_batch_object.status: return - if db_batch_object: + if effective_db_batch_object: verbose_proxy_logger.info( - "Updating batch %s status from %s to %s", batch_id, db_batch_object.status, response.status + "Updating batch %s status from %s to %s", batch_id, effective_db_batch_object.status, response.status ) else: verbose_proxy_logger.info("Updating batch %s status to %s after %s", batch_id, response.status, operation) diff --git a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py index c6f4128f2c5..db8f75cf640 100644 --- a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py +++ b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py @@ -414,7 +414,7 @@ async def test_batch_status_sync_from_provider_to_database(): # Verify logger was called with status change message mock_logger.info.assert_called() - log_message = mock_logger.info.call_args[0][0] + log_message = mock_logger.info.call_args[0][0] % mock_logger.info.call_args[0][1:] assert "validating" in log_message assert "completed" in log_message @@ -450,6 +450,9 @@ async def test_batch_cancel_updates_database(): # Mock prisma client mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( + return_value=None + ) mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() # Mock managed_files_obj @@ -482,7 +485,7 @@ async def test_batch_cancel_updates_database(): # Verify logger was called mock_logger.info.assert_called() - log_message = mock_logger.info.call_args[0][0] + log_message = mock_logger.info.call_args[0][0] % mock_logger.info.call_args[0][1:] assert "cancel" in log_message.lower() assert "cancelled" in log_message diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py index d8669960674..74139fa9238 100644 --- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py +++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py @@ -45,9 +45,10 @@ def _build_managed_files_mock(unified_id: str = "file-bWFuYWdlZF9vdXRwdXRfaWQ=") return mock -def _build_prisma_mock(): +def _build_prisma_mock(db_batch_object=None): mock = MagicMock() mock.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) + mock.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=db_batch_object) mock.db.litellm_managedobjecttable.update = AsyncMock() return mock @@ -89,6 +90,67 @@ async def test_update_batch_in_database_stores_unified_output_file_id(): assert stored["output_file_id"] != raw_output_file_id +@pytest.mark.asyncio +async def test_cancel_path_registers_output_file_under_batch_owner(): + unified_id = "file-bWFuYWdlZF9vdXRwdXRfaWQ=" + db_batch_object = SimpleNamespace( + created_by="batch-owner", team_id="batch-team", status="in_progress" + ) + response = _build_batch_response( + status="cancelling", + output_file_id="file-raw-output", + hidden_params={"model_id": "my-model", "model_name": "openai/gpt-4o"}, + ) + mock_managed_files = _build_managed_files_mock(unified_id=unified_id) + mock_prisma = _build_prisma_mock(db_batch_object=db_batch_object) + + await update_batch_in_database( + batch_id="batch_managed_ids_test", + unified_batch_id="litellm_proxy;model_id:my-model;llm_batch_id:batch_managed_ids_test", + response=response, + managed_files_obj=mock_managed_files, + prisma_client=mock_prisma, + verbose_proxy_logger=MagicMock(), + operation="cancel", + ) + + forwarded_auth = mock_managed_files.store_unified_file_id.call_args.kwargs[ + "user_api_key_dict" + ] + assert forwarded_auth.user_id == "batch-owner" + assert forwarded_auth.team_id == "batch-team" + stored = json.loads( + mock_prisma.db.litellm_managedobjecttable.update.call_args.kwargs["data"][ + "file_object" + ] + ) + assert stored["output_file_id"] == unified_id + + +@pytest.mark.asyncio +async def test_update_batch_derives_model_id_from_unified_batch_id(): + unified_id = "file-bWFuYWdlZF9vdXRwdXRfaWQ=" + response = _build_batch_response(output_file_id="file-raw-output", hidden_params={}) + mock_managed_files = _build_managed_files_mock(unified_id=unified_id) + mock_prisma = _build_prisma_mock() + + await update_batch_in_database( + batch_id="batch_managed_ids_test", + unified_batch_id="litellm_proxy;model_id:model-from-batch-id;llm_batch_id:batch_managed_ids_test", + response=response, + managed_files_obj=mock_managed_files, + prisma_client=mock_prisma, + verbose_proxy_logger=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(user_id="user-abc"), + ) + + assert ( + mock_managed_files.get_unified_output_file_id.call_args.kwargs["model_id"] + == "model-from-batch-id" + ) + assert response.output_file_id == unified_id + + @pytest.mark.asyncio async def test_ensure_batch_response_normalizes_error_file_id(): """Both output_file_id and error_file_id must be normalized to managed IDs.""" From 28a277e99e281052445e3568bcd1449a471447ba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:56:50 -0700 Subject: [PATCH 045/672] refactor(guardrails): drop dead tool extraction and an Any annotation, ratchet lint budgets --- basedpyright-code-budget.json | 10 +++++----- .../chat/guardrail_translation/handler.py | 16 ---------------- .../llms/base_llm/guardrail_translation/utils.py | 2 +- ruff-strict-budget.json | 2 +- type-discipline-budget.json | 6 +++--- 5 files changed, 10 insertions(+), 26 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 27d96e415fd..8a5c78c1f6c 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -3,7 +3,7 @@ "limit": 29204 }, "reportArgumentType": { - "limit": 2635 + "limit": 2634 }, "reportAssignmentType": { "limit": 329 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 9227 + "limit": 9226 }, "reportFunctionMemberAccess": { "limit": 7 @@ -105,7 +105,7 @@ "limit": 113 }, "reportUnknownMemberType": { - "limit": 40340 + "limit": 40339 }, "reportUnknownParameterType": { "limit": 20293 @@ -117,13 +117,13 @@ "limit": 122 }, "reportUnnecessaryComparison": { - "limit": 703 + "limit": 702 }, "reportUnnecessaryContains": { "limit": 5 }, "reportUnnecessaryIsInstance": { - "limit": 865 + "limit": 864 }, "reportUntypedBaseClass": { "limit": 72 diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index c25fa624f7f..60424fb78b5 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -574,22 +574,6 @@ class AnthropicMessagesHandler(BaseTranslation): data: Final = source.get("data") return (data,) if data else () - def _extract_input_tools( - self, - tools: list[dict[str, Any]], - tools_to_check: list[ChatCompletionToolParam], - ) -> None: - """ - Extract tools from a message. - """ - ## CHECK FOR TOOLS - if tools is not None and isinstance(tools, list): - # TRANSFORM ANTHROPIC TOOLS TO OPENAI TOOLS - openai_tools: Final = self.adapter.translate_anthropic_tools_to_openai( - tools=cast(list[AllAnthropicToolsValues], tools) - ) - tools_to_check.extend(openai_tools) - async def _apply_guardrail_responses_to_input( self, messages: list[dict[str, Any]], diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index fcd504fee2f..432ac64b456 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -130,7 +130,7 @@ def openai_messages_without_tool( return tuple(m for m in messages if _message_role(m) != "tool") -def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: Any) -> bool: +def effective_scan_only_tool_results_for_guardrail(guardrail_to_apply: object) -> bool: return getattr(guardrail_to_apply, "scan_only_tool_results", None) is True diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 421b424757b..ea20ac97e07 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -42,7 +42,7 @@ "limit": 81 }, "B010": { - "limit": 194 + "limit": 192 }, "B018": { "limit": 2 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index e26ce54ede7..37964c27657 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 23343 + "limit": 23337 }, "LIT002": { "limit": 27213 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1093 + "limit": 1092 }, "LIT007": { "limit": 0 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16802 + "limit": 16796 }, "LIT011": { "limit": 5602 From 6b5c7f92ce7b359ebb09ce69d9837bbaae1e3209 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:34:06 +0000 Subject: [PATCH 046/672] refactor(types): replace implicit and explicit Any across 11 modules Types the values that were flowing through as Any in the highest-density modules, using shapes the code already assumes: TypedDicts for the JSON payloads read by literal key, Protocols for the prisma rows, existing litellm types where they were already modeled, and `object` where a value is only stored and forwarded. Annotation-level only, no runtime behavior change. New annotations use read-only views (Mapping / Sequence / tuple) rather than dict / list, so LIT001 drops alongside the Any counts instead of trading one budget for another. No suppressions, casts, or type guards were added. basedpyright across the touched files: 1547 -> 856 errors, with reportAny down 399 and reportExplicitAny down 134, and no rule increasing. --- litellm/caching/redis_semantic_cache.py | 33 +++--- litellm/integrations/galileo.py | 83 +++++++++------ .../mcp_server/openapi_to_mcp_generator.py | 100 +++++++++++++----- .../claude_code_marketplace.py | 80 ++++++++++---- .../proxy/common_utils/custom_openapi_spec.py | 42 +++++--- .../proxy/container_endpoints/ownership.py | 80 ++++++++++---- .../cato_networks/cato_networks.py | 88 +++++++++++---- .../hiddenlayer/hiddenlayer.py | 45 ++++++-- .../guardrail_hooks/microsoft_purview/base.py | 61 ++++++----- .../workflow_management_endpoints.py | 98 ++++++++++++----- .../policy_engine/attachment_registry.py | 39 ++++--- 11 files changed, 516 insertions(+), 233 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index b1d298b79bb..604d6395ea1 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -13,6 +13,7 @@ import ast import asyncio import json import os +from collections.abc import Callable, Mapping from typing import Any, Final, cast import litellm @@ -47,7 +48,7 @@ class RedisSemanticCache(BaseCache): similarity_threshold: float | None = None, embedding_model: str = "text-embedding-ada-002", index_name: str | None = None, - **kwargs, + **kwargs: object, ): """ Initialize the Redis Semantic Cache. @@ -150,11 +151,11 @@ class RedisSemanticCache(BaseCache): def _init_semantic_cache( self, - semantic_cache_cls: Any, + semantic_cache_cls: Callable[..., object], index_name: str, redis_url: str, - cache_vectorizer: Any, - ) -> Any: + cache_vectorizer: object, + ) -> object: def _is_schema_mismatch(exc: ValueError) -> bool: error_message: Final = str(exc).lower() return any(phrase in error_message for phrase in ("schema does not match", "index schema")) @@ -206,12 +207,12 @@ class RedisSemanticCache(BaseCache): def _get_cache_filters(self, key: str) -> dict[str, str]: return {self.CACHE_KEY_FIELD_NAME: str(key)} - def _get_cache_key_filter_expression(self, key: str) -> Any: + def _get_cache_key_filter_expression(self, key: str) -> object: from redisvl.query.filter import Tag return Tag(self.CACHE_KEY_FIELD_NAME) == str(key) - def _cache_hit_matches_key(self, cache_hit: dict[str, Any], key: str) -> bool: + def _cache_hit_matches_key(self, cache_hit: Mapping[str, object], key: str) -> bool: # Pre-isolation entries with no ``litellm_cache_key`` field cannot be # safely reassigned to a caller's scope and are treated as misses. cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME) @@ -297,7 +298,7 @@ class RedisSemanticCache(BaseCache): return @staticmethod - def _coerce_response_input_value(value: Any) -> Any: + def _coerce_response_input_value(value: object) -> object: model_dump: Final = getattr(value, "model_dump", None) if callable(model_dump): return model_dump() @@ -340,7 +341,7 @@ class RedisSemanticCache(BaseCache): ) return embedding_response["data"][0]["embedding"] - def _get_cache_logic(self, cached_response: Any) -> Any: + def _get_cache_logic(self, cached_response: Any) -> object: """ Process the cached response to prepare it for use. @@ -369,7 +370,7 @@ class RedisSemanticCache(BaseCache): return cached_response - def set_cache(self, key: str, value: Any, **kwargs) -> None: + def set_cache(self, key: str, value: object, **kwargs) -> None: """ Store a value in the semantic cache. @@ -405,7 +406,7 @@ class RedisSemanticCache(BaseCache): except Exception as e: print_verbose(f"Error setting {value_str or value} in the Redis semantic cache: {e}") - def get_cache(self, key: str, **kwargs) -> Any: + def get_cache(self, key: str, **kwargs) -> object: """ Retrieve a semantically similar cached response. @@ -428,7 +429,7 @@ class RedisSemanticCache(BaseCache): # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. prompt_embedding: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) - check_kwargs: Final[dict[str, Any]] = { + check_kwargs: Final[Mapping[str, object]] = { "prompt": prompt, "vector": prompt_embedding, "filter_expression": self._get_cache_key_filter_expression(key), @@ -508,7 +509,7 @@ class RedisSemanticCache(BaseCache): print_verbose(f"Error generating async embedding: {e}") raise ValueError(f"Failed to generate embedding: {e}") from e - async def async_set_cache(self, key: str, value: Any, **kwargs) -> None: + async def async_set_cache(self, key: str, value: object, **kwargs) -> None: """ Asynchronously store a value in the semantic cache. @@ -548,7 +549,7 @@ class RedisSemanticCache(BaseCache): except Exception as e: print_verbose(f"Error in async_set_cache: {e}") - async def async_get_cache(self, key: str, **kwargs) -> Any: + async def async_get_cache(self, key: str, **kwargs) -> object: """ Asynchronously retrieve a semantically similar cached response. @@ -573,7 +574,7 @@ class RedisSemanticCache(BaseCache): # Check the cache for semantically similar prompts in this exact # LiteLLM cache-key scope. - check_kwargs: Final[dict[str, Any]] = { + check_kwargs: Final[Mapping[str, object]] = { "prompt": prompt, "vector": prompt_embedding, "filter_expression": self._get_cache_key_filter_expression(key), @@ -615,7 +616,7 @@ class RedisSemanticCache(BaseCache): print_verbose(f"Error in async_get_cache: {e}") kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0 - async def _index_info(self) -> dict[str, Any]: + async def _index_info(self) -> Mapping[str, object]: """ Get information about the Redis index. @@ -625,7 +626,7 @@ class RedisSemanticCache(BaseCache): aindex: Final = await self.llmcache._get_async_index() return await aindex.info() - async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs) -> None: + async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs: object) -> None: """ Asynchronously store multiple values in the semantic cache. diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index 12c2ac8a53f..f9ec825e922 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -4,7 +4,8 @@ import json import os import re import uuid -from datetime import datetime, timezone +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone, tzinfo from typing import Any, Final, cast import httpx @@ -59,7 +60,7 @@ class LLMResponse(BaseModel): class GalileoObserve(CustomLogger): def __init__(self) -> None: - self.in_memory_records: list[dict] = [] + self.in_memory_records: list[dict[str, Any]] = [] self.batch_size = 1 self.api_key = os.getenv("GALILEO_API_KEY") self.project_id = os.getenv("GALILEO_PROJECT_ID") @@ -176,7 +177,7 @@ class GalileoObserve(CustomLogger): return False @staticmethod - def _galileo_input_messages(messages: Any | None, input_text: str) -> list[dict[str, str]]: + def _galileo_input_messages(messages: object, input_text: str) -> list[dict[str, str]]: if isinstance(messages, dict): messages = messages.get("messages") if not messages: @@ -203,11 +204,11 @@ class GalileoObserve(CustomLogger): return [{"role": "user", "content": input_text}] @staticmethod - def _local_timezone(): + def _local_timezone() -> tzinfo: return datetime.now().astimezone().tzinfo or timezone.utc @staticmethod - def _format_created_at(dt: datetime | Any) -> str: + def _format_created_at(dt: object) -> str: """Serialize timestamps as UTC ISO-8601 for Galileo.""" if not isinstance(dt, datetime): return str(dt) @@ -226,7 +227,7 @@ class GalileoObserve(CustomLogger): return created_at @staticmethod - def _token_metrics_from_record(record: dict[str, Any]) -> dict[str, Any]: + def _token_metrics_from_record(record: Mapping[str, Any]) -> dict[str, Any]: num_input_tokens: Final = int(record.get("num_input_tokens") or 0) num_output_tokens: Final = int(record.get("num_output_tokens") or 0) num_total_tokens = int(record.get("num_total_tokens") or 0) @@ -244,7 +245,7 @@ class GalileoObserve(CustomLogger): @staticmethod def _record_to_v2_span( - record: dict[str, Any], + record: Mapping[str, Any], *, trace_id: str, span_id: str, @@ -275,7 +276,7 @@ class GalileoObserve(CustomLogger): return span @staticmethod - def _record_to_v2_trace(record: dict[str, Any]) -> dict[str, Any]: + def _record_to_v2_trace(record: Mapping[str, Any]) -> dict[str, Any]: trace_id: Final = str(uuid.uuid4()) span_id: Final = str(uuid.uuid4()) created_at: Final = GalileoObserve._normalize_created_at(record.get("created_at", "")) @@ -295,7 +296,7 @@ class GalileoObserve(CustomLogger): "spans": [GalileoObserve._record_to_v2_span(record, trace_id=trace_id, span_id=span_id)], } - def _build_traces_payload(self, records: list[dict]) -> dict[str, Any]: + def _build_traces_payload(self, records: Sequence[Mapping[str, Any]]) -> dict[str, Any]: payload: Final[dict[str, Any]] = { "traces": [self._record_to_v2_trace(record) for record in records], "logging_method": "api_direct", @@ -357,7 +358,7 @@ class GalileoObserve(CustomLogger): @staticmethod def _log_v2_payload_validation(payload: dict[str, Any]) -> None: missing_fields: Final[list[str]] = [] - traces: Final = payload.get("traces", []) + traces: Final[Sequence[object]] = payload.get("traces", []) if not traces: missing_fields.append("traces") @@ -385,7 +386,7 @@ class GalileoObserve(CustomLogger): ) def _log_flush_payload(self, url: str, payload: dict[str, Any]) -> None: - traces: Final = payload.get("traces", []) + traces: Final[Sequence[object]] = payload.get("traces", []) verbose_logger.debug( "Galileo Logger flush URL: %s trace_count=%s", url, @@ -415,8 +416,8 @@ class GalileoObserve(CustomLogger): pass @staticmethod - def _build_prompt(kwargs: dict[str, Any]) -> dict[str, Any]: - optional_params: Final = kwargs.get("optional_params", {}) or {} + def _build_prompt(kwargs: Mapping[str, Any]) -> dict[str, Any]: + optional_params: Final[Mapping[str, object]] = kwargs.get("optional_params", {}) or {} prompt: Final[dict[str, Any]] = {"messages": kwargs.get("messages")} if optional_params.get("functions") is not None: prompt["functions"] = optional_params["functions"] @@ -425,13 +426,13 @@ class GalileoObserve(CustomLogger): return prompt @staticmethod - def _serialize_galileo_output(value: Any) -> str: + def _serialize_galileo_output(value: object) -> str: if value is None: return "" if isinstance(value, str): return value - def _json_default(obj: Any) -> Any: + def _json_default(obj: Any) -> object: if hasattr(obj, "model_dump"): return obj.model_dump() return str(obj) @@ -439,8 +440,8 @@ class GalileoObserve(CustomLogger): return json.dumps(value, default=_json_default) @staticmethod - def _prompt_to_input_text(prompt: dict[str, Any]) -> str: - messages: Final = prompt.get("messages") + def _prompt_to_input_text(prompt: Mapping[str, Any]) -> str: + messages: Final[object] = prompt.get("messages") if messages is not None: text: Final = GalileoObserve._input_text_from_messages(messages) if text: @@ -448,7 +449,7 @@ class GalileoObserve(CustomLogger): return json.dumps(prompt, default=str) @staticmethod - def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> Any: + def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> object: if response_obj.choices and len(response_obj.choices) > 0: message: Final = response_obj["choices"][0]["message"] if hasattr(message, "json"): @@ -470,23 +471,23 @@ class GalileoObserve(CustomLogger): @staticmethod def _get_responses_api_content_for_galileo( response_obj: ResponsesAPIResponse, - ) -> Any: + ) -> object: if hasattr(response_obj, "output") and response_obj.output: return response_obj.output return None @staticmethod - def _langfuse_style_rerank_prompt(kwargs: dict[str, Any]) -> dict[str, Any]: + def _langfuse_style_rerank_prompt(kwargs: Mapping[str, object]) -> dict[str, Any]: """Match Langfuse rerank input: prompt = {"messages": kwargs.get("messages")}.""" return {"messages": kwargs.get("messages")} def _get_galileo_input_output_content( self, - kwargs: dict[str, Any], - response_obj: Any, + kwargs: Mapping[str, object], + response_obj: object, level: str = "DEFAULT", status_message: str | None = None, - ) -> tuple[str, str, Any]: + ) -> tuple[str, str, object]: """ Mirror Langfuse _get_langfuse_input_output_content for Galileo ingest. @@ -582,12 +583,12 @@ class GalileoObserve(CustomLogger): return self._prompt_to_input_text(prompt), "", kwargs.get("messages") or [] - def get_output_str_from_response(self, response_obj: Any, kwargs: dict[str, Any]) -> str: + def get_output_str_from_response(self, response_obj: object, kwargs: Mapping[str, object]) -> str: _, output_text, _ = self._get_galileo_input_output_content(kwargs=kwargs, response_obj=response_obj) return output_text @staticmethod - def _input_text_from_messages(messages: Any) -> str: + def _input_text_from_messages(messages: object) -> str: """Return a plain-string summary of the input suitable for the trace-level input field.""" if isinstance(messages, str): return messages @@ -613,7 +614,13 @@ class GalileoObserve(CustomLogger): return str(content) return "" - async def async_log_success_event(self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any): + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: verbose_logger.debug("On Async Success") try: await self._async_log_success_event_impl( @@ -625,7 +632,13 @@ class GalileoObserve(CustomLogger): except Exception: verbose_logger.exception("Galileo Logger: unexpected error in async_log_success_event") - async def _async_log_success_event_impl(self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any): + async def _async_log_success_event_impl( + self, + kwargs: Mapping[str, Any], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: if not self._is_configured(): verbose_logger.debug( "Galileo Logger: skipping — GALILEO_PROJECT_ID=%s GALILEO_API_KEY=%s GALILEO_BASE_URL=%s", @@ -635,7 +648,7 @@ class GalileoObserve(CustomLogger): ) return - slo: Final[dict[str, Any] | None] = kwargs.get("standard_logging_object") + slo: Final[Mapping[str, Any] | None] = kwargs.get("standard_logging_object") if slo is None: verbose_logger.debug("Galileo Logger: no standard_logging_object in kwargs, skipping") return @@ -646,8 +659,8 @@ class GalileoObserve(CustomLogger): kwargs=kwargs, response_obj=response_obj ) - raw_start: Final = slo.get("startTime") - raw_end: Final = slo.get("endTime") + raw_start: Final[float | None] = slo.get("startTime") + raw_end: Final[float | None] = slo.get("endTime") if raw_start is None or raw_end is None: verbose_logger.debug( "Galileo Logger: standard_logging_object missing startTime/endTime, " @@ -710,7 +723,7 @@ class GalileoObserve(CustomLogger): if len(self.in_memory_records) >= self.batch_size: await self.flush_in_memory_records() - async def flush_in_memory_records(self): + async def flush_in_memory_records(self) -> None: if not self.in_memory_records: return @@ -774,5 +787,11 @@ class GalileoObserve(CustomLogger): if not self.use_v2_api and response.status_code in (401, 403): self.headers = None - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: verbose_logger.debug("On Async Failure") diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 1794a1b66b8..41057840684 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -7,10 +7,13 @@ import contextvars import json import os import re +from collections.abc import Mapping, Sequence from pathlib import PurePosixPath -from typing import Any, Final +from typing import Any, Final, TypeAlias, TypedDict from urllib.parse import quote +import httpx + # Tool names emitted from OpenAPI specs must work across all major LLM providers. # OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to # ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use @@ -44,6 +47,43 @@ from litellm.proxy._experimental.mcp_server.tool_registry import ( global_mcp_tool_registry, ) +_OpenAPIObject: TypeAlias = Mapping[str, Any] + + +class _OpenAPIParameterSchema(TypedDict, total=False): + type: str + + +class _OpenAPIJSONSchema(TypedDict, total=False): + properties: Mapping[str, object] + + +class _OpenAPIMediaType(TypedDict, total=False): + schema: _OpenAPIJSONSchema + + +class _OpenAPIRequestBody(TypedDict, total=False): + description: str + required: bool + content: Mapping[str, _OpenAPIMediaType] + + +class _OpenAPIOperation(TypedDict, total=False): + operationId: str + summary: str + description: str + parameters: Sequence[_OpenAPIObject] + requestBody: _OpenAPIRequestBody + + +class _OpenAPIPathItem(TypedDict, total=False): + parameters: Sequence[_OpenAPIObject] + + +class _OpenAPIComponents(TypedDict, total=False): + parameters: Mapping[str, _OpenAPIObject] + + # Store the base URL and headers globally BASE_URL: Final = "" HEADERS: Final[dict[str, str]] = {} @@ -69,7 +109,7 @@ _request_resolved_auth_headers: Final[contextvars.ContextVar[dict[str, str] | No ) -def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str: +def _sanitize_path_parameter_value(param_value: object, param_name: str) -> str: """Ensure path params cannot introduce directory traversal.""" if param_value is None: return "" @@ -109,7 +149,7 @@ def load_openapi_spec(filepath: str) -> dict[str, Any]: async def load_openapi_spec_async(filepath: str) -> dict[str, Any]: if filepath.startswith("http://") or filepath.startswith("https://"): client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) - r: Final = await async_safe_get(client, filepath) + r: Final[httpx.Response] = await async_safe_get(client, filepath) r.raise_for_status() return r.json() @@ -121,11 +161,11 @@ async def load_openapi_spec_async(filepath: str) -> dict[str, Any]: return json.load(f) -def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: +def get_base_url(spec: _OpenAPIObject, spec_path: str | None = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: - server_url: Final = spec["servers"][0]["url"] + server_url: Final[str] = spec["servers"][0]["url"] # If the server URL is relative (starts with /), derive base from spec_path if server_url.startswith("/") and spec_path: @@ -147,8 +187,8 @@ def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: return server_url # OpenAPI 2.x (Swagger) elif "host" in spec: - scheme: Final = spec.get("schemes", ["https"])[0] - base_path: Final = spec.get("basePath", "") + scheme: Final[str] = spec.get("schemes", ["https"])[0] + base_path: Final[str] = spec.get("basePath", "") return f"{scheme}://{spec['host']}{base_path}" # Fallback: derive base URL from spec_path if it's a URL @@ -172,20 +212,22 @@ def get_base_url(spec: dict[str, Any], spec_path: str | None = None) -> str: return "" -def _resolve_ref(param: dict[str, Any], component_params: dict[str, Any]) -> dict[str, Any] | None: +def _resolve_ref(param: _OpenAPIObject, component_params: Mapping[str, _OpenAPIObject]) -> _OpenAPIObject | None: """Resolve a single parameter, following a $ref if present. Returns the resolved param dict, or None if the $ref target is absent from components (so callers can skip/filter it rather than propagating a stub with name=None that would corrupt deduplication). """ - ref: Final = param.get("$ref", "") + ref: Final[str] = param.get("$ref", "") if not ref.startswith("#/components/parameters/"): return param return component_params.get(ref.split("/")[-1]) -def _resolve_param_list(raw: list[dict[str, Any]], component_params: dict[str, Any]) -> list[dict[str, Any]]: +def _resolve_param_list( + raw: Sequence[_OpenAPIObject], component_params: Mapping[str, _OpenAPIObject] +) -> list[_OpenAPIObject]: """Resolve $refs in a parameter list, dropping any unresolvable entries.""" result: Final = [] for p in raw: @@ -196,9 +238,9 @@ def _resolve_param_list(raw: list[dict[str, Any]], component_params: dict[str, A def resolve_operation_params( - operation: dict[str, Any], - path_item: dict[str, Any], - components: dict[str, Any], + operation: _OpenAPIOperation, + path_item: _OpenAPIPathItem, + components: _OpenAPIComponents, ) -> dict[str, Any]: """Return a copy of *operation* with fully-resolved, merged parameters. @@ -214,7 +256,7 @@ def resolve_operation_params( merged with the operation-level params; operation-level wins when the same ``name`` + ``in`` combination appears in both. """ - component_params: Final = components.get("parameters", {}) + component_params: Final[Mapping[str, _OpenAPIObject]] = components.get("parameters", {}) path_level: Final = _resolve_param_list(path_item.get("parameters", []), component_params) op_level: Final = _resolve_param_list(operation.get("parameters", []), component_params) op_keys: Final = {(p["name"], p.get("in")) for p in op_level} @@ -224,8 +266,9 @@ def resolve_operation_params( return result -def extract_parameters(operation: dict[str, Any]) -> tuple: +def extract_parameters(operation: _OpenAPIObject) -> tuple[Sequence[str], Sequence[str], Sequence[str]]: """Extract parameter names from OpenAPI operation.""" + param: _OpenAPIObject path_params: Final = [] query_params: Final = [] body_params: Final = [] @@ -235,7 +278,7 @@ def extract_parameters(operation: dict[str, Any]) -> tuple: for param in operation["parameters"]: if "name" not in param: continue - param_name = param["name"] + param_name: str = param["name"] if param.get("in") == "path": path_params.append(param_name) elif param.get("in") == "query": @@ -250,8 +293,9 @@ def extract_parameters(operation: dict[str, Any]) -> tuple: return path_params, query_params, body_params -def build_input_schema(operation: dict[str, Any]) -> dict[str, Any]: +def build_input_schema(operation: _OpenAPIObject) -> dict[str, Any]: """Build MCP input schema from OpenAPI operation.""" + param: _OpenAPIObject properties: Final = {} required: Final = [] @@ -260,9 +304,9 @@ def build_input_schema(operation: dict[str, Any]) -> dict[str, Any]: for param in operation["parameters"]: if "name" not in param: continue - param_name = param["name"] - param_schema = param.get("schema", {}) - param_type = param_schema.get("type", "string") + param_name: str = param["name"] + param_schema: _OpenAPIParameterSchema = param.get("schema", {}) + param_type: str = param_schema.get("type", "string") properties[param_name] = { "type": param_type, @@ -274,12 +318,12 @@ def build_input_schema(operation: dict[str, Any]) -> dict[str, Any]: # Process requestBody (OpenAPI 3.x) if "requestBody" in operation: - request_body: Final = operation["requestBody"] - content: Final = request_body.get("content", {}) + request_body: Final[_OpenAPIRequestBody] = operation["requestBody"] + content: Final[Mapping[str, _OpenAPIMediaType]] = request_body.get("content", {}) # Try to get JSON schema if "application/json" in content: - schema: Final = content["application/json"].get("schema", {}) + schema: Final[_OpenAPIJSONSchema] = content["application/json"].get("schema", {}) properties["body"] = { "type": "object", "description": request_body.get("description", "Request body"), @@ -347,7 +391,7 @@ def _merge_openapi_tool_request_headers( def create_tool_function( path: str, method: str, - operation: dict[str, Any], + operation: _OpenAPIObject, base_url: str, headers: dict[str, str] | None = None, ): @@ -373,7 +417,7 @@ def create_tool_function( path_params, query_params, body_params = extract_parameters(operation) original_method: Final = method.lower() - async def tool_function(**kwargs: Any) -> str: + async def tool_function(**kwargs: object) -> str: """ Dynamically generated tool function. @@ -448,10 +492,10 @@ def create_tool_function( return tool_function -def register_tools_from_openapi(spec: dict[str, Any], base_url: str): +def register_tools_from_openapi(spec: _OpenAPIObject, base_url: str) -> None: """Register MCP tools from OpenAPI specification.""" - paths: Final = spec.get("paths", {}) - used_names: Final[set] = set() + paths: Final[Mapping[str, Mapping[str, _OpenAPIOperation]]] = spec.get("paths", {}) + used_names: Final = set() for path, path_item in paths.items(): for method in ["get", "post", "put", "delete", "patch"]: diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 579c735b180..d340ca2ced3 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -18,8 +18,9 @@ Endpoints: import json import re +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Any, Final +from typing import Final, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import JSONResponse @@ -41,7 +42,30 @@ from litellm.types.proxy.claude_code_endpoints import ( router: Final = APIRouter() -async def _get_prisma_client(): +class _PluginRecord(Protocol): + id: str + name: str + version: str | None + description: str | None + manifest_json: str + enabled: bool + created_at: datetime | None + updated_at: datetime | None + created_by: str | None + + +class _MarketplaceEntry(TypedDict, total=False): + name: str + source: object + version: str + description: str + author: object + homepage: object + keywords: object + category: object + + +async def _get_prisma_client() -> object: """Get the prisma client from proxy_server.""" from litellm.proxy.proxy_server import prisma_client @@ -77,12 +101,14 @@ async def get_marketplace(): try: prisma_client: Final = await _get_prisma_client() - plugins: Final = await ClaudeCodePluginRepository(prisma_client).table.find_many(where={"enabled": True}) + plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( + where={"enabled": True} + ) plugin_list: Final = [] for plugin in plugins: try: - manifest = json.loads(plugin.manifest_json) + manifest: Mapping[str, object] = json.loads(plugin.manifest_json) except json.JSONDecodeError: verbose_proxy_logger.warning("Plugin %s has invalid manifest JSON, skipping", plugin.name) continue @@ -92,7 +118,7 @@ async def get_marketplace(): verbose_proxy_logger.warning("Plugin %s has no source field, skipping", plugin.name) continue - entry: dict[str, Any] = { + entry: _MarketplaceEntry = { "name": plugin.name, "source": manifest["source"], } @@ -137,7 +163,7 @@ async def get_marketplace(): _VALID_GIT_SUBDIR_PATH_RE: Final = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]*(/[a-zA-Z0-9][a-zA-Z0-9._-]*)*$") -def _validate_plugin_source(source: dict[str, Any]) -> None: +def _validate_plugin_source(source: Mapping[str, str]) -> None: """Validate plugin source format, raising HTTPException on invalid input.""" source_type: Final = source.get("source") if source_type == "github": @@ -179,9 +205,9 @@ def _validate_plugin_source(source: dict[str, Any]) -> None: ) -def _build_plugin_manifest(name: str, spec: PluginSpec) -> dict[str, Any]: +def _build_plugin_manifest(name: str, spec: PluginSpec) -> Mapping[str, object]: """Build the stored manifest dict shared by plugin create and update.""" - dumped = spec.model_dump(exclude_none=True) + dumped: Final[Mapping[str, object]] = spec.model_dump(exclude_none=True) return {"name": name, **{key: value for key, value in dumped.items() if value and key != "name"}} @@ -255,14 +281,16 @@ async def register_plugin( _validate_plugin_source(request.source) - existing = await ClaudeCodePluginRepository(prisma_client).table.find_unique(where={"name": request.name}) + existing: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( + where={"name": request.name} + ) if existing: raise _name_conflict_error(request.name) - manifest = _build_plugin_manifest(request.name, request) + manifest: Final[Mapping[str, object]] = _build_plugin_manifest(request.name, request) try: - plugin = await ClaudeCodePluginRepository(prisma_client).table.create( + plugin: Final[_PluginRecord] = await ClaudeCodePluginRepository(prisma_client).table.create( data={ "name": request.name, "version": request.version, @@ -326,7 +354,9 @@ async def list_plugins( prisma_client: Final = await _get_prisma_client() where: Final = {"enabled": True} if enabled_only else {} - plugins: Final = await ClaudeCodePluginRepository(prisma_client).table.find_many(where=where) + plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many( + where=where + ) plugin_list: Final = [] for p in plugins: @@ -391,7 +421,9 @@ async def get_plugin( try: prisma_client: Final = await _get_prisma_client() - plugin: Final = await ClaudeCodePluginRepository(prisma_client).table.find_unique(where={"name": plugin_name}) + plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( + where={"name": plugin_name} + ) if not plugin: raise HTTPException( @@ -399,7 +431,7 @@ async def get_plugin( detail={"error": f"Plugin '{plugin_name}' not found"}, ) - manifest: Final = json.loads(plugin.manifest_json) if plugin.manifest_json else {} + manifest: Final[Mapping[str, object]] = json.loads(plugin.manifest_json) if plugin.manifest_json else {} return { "id": plugin.id, @@ -477,19 +509,19 @@ async def update_plugin( from prisma.errors import PrismaError try: - prisma_client = await _get_prisma_client() + prisma_client: Final = await _get_prisma_client() _validate_plugin_source(request.source) - existing = await ClaudeCodePluginRepository(prisma_client).table.find_unique( + existing: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( where={"name": plugin_name} # mutable-ok: prisma query arguments must be plain dicts ) if not existing: raise _error_response(404, f"Plugin '{plugin_name}' not found") - manifest = _build_plugin_manifest(plugin_name, request) + manifest: Final[Mapping[str, object]] = _build_plugin_manifest(plugin_name, request) - plugin = await ClaudeCodePluginRepository(prisma_client).table.update( + plugin: Final[_PluginRecord] = await ClaudeCodePluginRepository(prisma_client).table.update( where={"name": plugin_name}, # mutable-ok: prisma query arguments must be plain dicts data={ # mutable-ok: prisma query arguments must be plain dicts "version": request.version, @@ -540,7 +572,9 @@ async def enable_plugin( try: prisma_client: Final = await _get_prisma_client() - plugin: Final = await ClaudeCodePluginRepository(prisma_client).table.find_unique(where={"name": plugin_name}) + plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( + where={"name": plugin_name} + ) if not plugin: raise HTTPException( status_code=404, @@ -583,7 +617,9 @@ async def disable_plugin( try: prisma_client: Final = await _get_prisma_client() - plugin: Final = await ClaudeCodePluginRepository(prisma_client).table.find_unique(where={"name": plugin_name}) + plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( + where={"name": plugin_name} + ) if not plugin: raise HTTPException( status_code=404, @@ -626,7 +662,9 @@ async def delete_plugin( try: prisma_client: Final = await _get_prisma_client() - plugin: Final = await ClaudeCodePluginRepository(prisma_client).table.find_unique(where={"name": plugin_name}) + plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique( + where={"name": plugin_name} + ) if not plugin: raise HTTPException( status_code=404, diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index cb91805aafc..3bedb6f720c 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -1,8 +1,14 @@ -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Any, Final, TypedDict from litellm._logging import verbose_proxy_logger +class _FieldSchema(TypedDict, total=False): + type: str + anyOf: Sequence["_FieldSchema"] + + class CustomOpenAPISpec: """ Handler for customizing OpenAPI specifications with Pydantic models @@ -26,7 +32,7 @@ class CustomOpenAPISpec: RESPONSES_API_PATHS = ["/v1/responses", "/responses"] @staticmethod - def get_pydantic_schema(model_class) -> dict[str, Any] | None: + def get_pydantic_schema(model_class) -> Mapping[str, object] | None: """ Get JSON schema from a Pydantic model, handling both v1 and v2 APIs. @@ -53,7 +59,9 @@ class CustomOpenAPISpec: return None @staticmethod - def add_schema_to_components(openapi_schema: dict[str, Any], schema_name: str, schema_def: dict[str, Any]) -> None: + def add_schema_to_components( + openapi_schema: dict[str, Any], schema_name: str, schema_def: Mapping[str, object] + ) -> None: """ Add a schema definition to the OpenAPI components/schemas section. @@ -72,7 +80,7 @@ class CustomOpenAPISpec: CustomOpenAPISpec._move_defs_to_components(openapi_schema, {schema_name: schema_def}) @staticmethod - def add_request_body_to_paths(openapi_schema: dict[str, Any], paths: list[str], schema_ref: str) -> None: + def add_request_body_to_paths(openapi_schema: dict[str, Any], paths: Sequence[str], schema_ref: str) -> None: """ Add request body with expanded form fields for better Swagger UI display. This keeps the request body but expands it to show individual fields in the UI. @@ -130,7 +138,7 @@ class CustomOpenAPISpec: openapi_schema["paths"][path]["post"]["parameters"] = filtered_params @staticmethod - def _move_defs_to_components(openapi_schema: dict[str, Any], defs: dict[str, Any]) -> None: + def _move_defs_to_components(openapi_schema: dict[str, Any], defs: Mapping[str, Mapping[str, Any]]) -> None: """ Move $defs from Pydantic v2 schema to OpenAPI components/schemas. This makes the definitions resolvable in Swagger/OpenAPI viewers. @@ -190,7 +198,7 @@ class CustomOpenAPISpec: return schema @staticmethod - def _extract_field_schema(field_def: dict[str, Any]) -> dict[str, Any]: + def _extract_field_schema(field_def: _FieldSchema) -> _FieldSchema: """ Extract a simple schema from a Pydantic field definition for parameter display. @@ -218,7 +226,7 @@ class CustomOpenAPISpec: return {"type": "string"} @staticmethod - def _expand_field_definition(field_def: dict[str, Any]) -> dict[str, Any]: + def _expand_field_definition(field_def: dict[str, object]) -> dict[str, object]: """ Expand a Pydantic field definition for inline use in OpenAPI schema. This creates a full field definition that Swagger UI can render as individual form fields. @@ -234,12 +242,12 @@ class CustomOpenAPISpec: @staticmethod def add_request_schema( - openapi_schema: dict[str, Any], + openapi_schema: dict[str, object], model_class: type, schema_name: str, - paths: list[str], + paths: Sequence[str], operation_name: str, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Generic method to add a request schema to OpenAPI specification. @@ -279,8 +287,8 @@ class CustomOpenAPISpec: @staticmethod def add_chat_completion_request_schema( - openapi_schema: dict[str, Any], - ) -> dict[str, Any]: + openapi_schema: dict[str, object], + ) -> dict[str, object]: """ Add ProxyChatCompletionRequest schema to chat completion endpoints for documentation. This shows the request body in Swagger without runtime validation. @@ -306,7 +314,7 @@ class CustomOpenAPISpec: return openapi_schema @staticmethod - def add_embedding_request_schema(openapi_schema: dict[str, Any]) -> dict[str, Any]: + def add_embedding_request_schema(openapi_schema: dict[str, object]) -> dict[str, object]: """ Add EmbeddingRequest schema to embedding endpoints for documentation. This shows the request body in Swagger without runtime validation. @@ -333,8 +341,8 @@ class CustomOpenAPISpec: @staticmethod def add_responses_api_request_schema( - openapi_schema: dict[str, Any], - ) -> dict[str, Any]: + openapi_schema: dict[str, object], + ) -> dict[str, object]: """ Add ResponsesAPIRequestParams schema to responses API endpoints for documentation. This shows the request body in Swagger without runtime validation. @@ -361,8 +369,8 @@ class CustomOpenAPISpec: @staticmethod def add_llm_api_request_schema_body( - openapi_schema: dict[str, Any], - ) -> dict[str, Any]: + openapi_schema: dict[str, object], + ) -> dict[str, object]: """ Add LLM API request schema bodies to OpenAPI specification for documentation. diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py index e582ddb36c7..c40f4d4fef2 100644 --- a/litellm/proxy/container_endpoints/ownership.py +++ b/litellm/proxy/container_endpoints/ownership.py @@ -1,5 +1,7 @@ import json -from typing import Any, Final +from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet +from typing import TYPE_CHECKING, Any, Final, Protocol from fastapi import HTTPException @@ -15,6 +17,36 @@ from litellm.proxy.common_utils.resource_ownership import ( from litellm.repositories.table_repositories import ManagedObjectRepository from litellm.responses.utils import ResponsesAPIRequestUtils +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _ManagedObjectRow(Protocol): + model_object_id: str + unified_object_id: str | None + file_purpose: str | None + created_by: str | None + + +class _ManagedObjectTable(Protocol): + async def find_unique(self, *, where: Mapping[str, str]) -> _ManagedObjectRow | None: ... + + async def find_first(self, *, where: Mapping[str, str]) -> _ManagedObjectRow | None: ... + + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ManagedObjectRow]: ... + + async def create(self, *, data: Mapping[str, str]) -> _ManagedObjectRow: ... + + async def update(self, *, where: Mapping[str, str], data: Mapping[str, str]) -> _ManagedObjectRow | None: ... + + +class _ContainerListResponse(Protocol): + data: Sequence[object] + first_id: str | None + last_id: str | None + has_more: bool + + CONTAINER_OBJECT_PURPOSE: Final = "container" # 60s LRU/TTL cache absorbs every container access check before it reaches @@ -39,7 +71,7 @@ _CONTAINER_STORED_ID_CACHE: Final = InMemoryCache(max_size_in_memory=10000, defa _ALLOWED_CONTAINER_IDS_CACHE: Final = InMemoryCache(max_size_in_memory=2048, default_ttl=60) -def _allowed_container_ids_cache_key(owner_scopes: list[str]) -> str: +def _allowed_container_ids_cache_key(owner_scopes: Sequence[str]) -> str: """JSON-encode the sorted scope list — using a separator like ``|`` would collide for any tenant whose user_id / team_id / org_id / api_key happens to contain the separator. JSON quoting escapes @@ -86,7 +118,7 @@ async def get_container_forwarding_params( return params -def _get_response_id(response: Any) -> str | None: +def _get_response_id(response: object) -> str | None: if response is None: return None if isinstance(response, dict): @@ -96,7 +128,7 @@ def _get_response_id(response: Any) -> str | None: return value if isinstance(value, str) else None -def _dump_response(response: Any) -> dict[str, Any]: +def _dump_response(response: Any) -> dict[str, object]: if isinstance(response, dict): return dict(response) if hasattr(response, "model_dump"): @@ -106,17 +138,17 @@ def _dump_response(response: Any) -> dict[str, Any]: return {"id": _get_response_id(response)} -async def _get_prisma_client(): +async def _get_prisma_client() -> "PrismaClient | None": from litellm.proxy.proxy_server import prisma_client return prisma_client def _custom_llm_provider_from_responses_response( - response: Any, + response: object, default: str = "openai", ) -> str: - hidden_params: dict[str, Any] = {} + hidden_params: Mapping[str, object] = {} if isinstance(response, dict): hidden_params = response.get("_hidden_params") or {} else: @@ -129,7 +161,7 @@ def _custom_llm_provider_from_responses_response( async def record_container_owners_from_responses_response( - response: Any, + response: object, user_api_key_dict: UserAPIKeyAuth, custom_llm_provider: str | None = None, ) -> None: @@ -160,10 +192,10 @@ async def record_container_owners_from_responses_response( async def record_container_owner( - response: Any, + response: object, user_api_key_dict: UserAPIKeyAuth, custom_llm_provider: str, -) -> Any: +) -> object: container_id: Final = _get_response_id(response) if container_id is None: verbose_proxy_logger.warning("Skipping container ownership tracking because provider response has no id") @@ -195,7 +227,7 @@ async def record_container_owner( verbose_proxy_logger.warning("Skipping container ownership tracking because prisma_client is None") return response - table: Final = ManagedObjectRepository(prisma_client).table + table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table existing: Final = await table.find_unique(where={"model_object_id": model_object_id}) if existing is not None: if getattr(existing, "file_purpose", None) != CONTAINER_OBJECT_PURPOSE: @@ -247,15 +279,15 @@ async def _get_container_owner(original_container_id: str, custom_llm_provider: if prisma_client is None: return None - row: Final = await ManagedObjectRepository(prisma_client).table.find_first( + row: Final[_ManagedObjectRow | None] = await ManagedObjectRepository(prisma_client).table.find_first( where={ "model_object_id": model_object_id, "file_purpose": CONTAINER_OBJECT_PURPOSE, } ) - owner: Final = getattr(row, "created_by", None) if row is not None else None + owner: Final[str | None] = getattr(row, "created_by", None) if row is not None else None _CONTAINER_OWNER_CACHE.set_cache(model_object_id, owner if owner is not None else _NEGATIVE_OWNER_SENTINEL) - stored_id: Final = getattr(row, "unified_object_id", None) if row is not None else None + stored_id: Final[str | None] = getattr(row, "unified_object_id", None) if row is not None else None _CONTAINER_STORED_ID_CACHE.set_cache( model_object_id, (stored_id if isinstance(stored_id, str) and stored_id else _NEGATIVE_STORED_ID_SENTINEL), @@ -283,13 +315,13 @@ async def _get_stored_container_id(original_container_id: str, custom_llm_provid if prisma_client is None: return None - row: Final = await ManagedObjectRepository(prisma_client).table.find_first( + row: Final[_ManagedObjectRow | None] = await ManagedObjectRepository(prisma_client).table.find_first( where={ "model_object_id": model_object_id, "file_purpose": CONTAINER_OBJECT_PURPOSE, } ) - stored_id: Final = getattr(row, "unified_object_id", None) if row is not None else None + stored_id: Final[str | None] = getattr(row, "unified_object_id", None) if row is not None else None _CONTAINER_STORED_ID_CACHE.set_cache( model_object_id, (stored_id if isinstance(stored_id, str) and stored_id else _NEGATIVE_STORED_ID_SENTINEL), @@ -317,7 +349,7 @@ async def assert_user_can_access_container( return original_container_id, resolved_provider -def _get_container_list_data(response: Any) -> list[Any] | None: +def _get_container_list_data(response: object) -> Sequence[object] | None: if response is None: return None if isinstance(response, dict): @@ -327,7 +359,9 @@ def _get_container_list_data(response: Any) -> list[Any] | None: return data if isinstance(data, list) else None -def _set_container_list_data(response: Any, data: list[Any], removed_filtered_items: bool = False) -> Any: +def _set_container_list_data( + response: _ContainerListResponse, data: list[object], removed_filtered_items: bool = False +) -> _ContainerListResponse: if isinstance(response, dict): response["data"] = data if data: @@ -353,7 +387,7 @@ def _set_container_list_data(response: Any, data: list[Any], removed_filtered_it async def _get_allowed_container_ids( user_api_key_dict: UserAPIKeyAuth, -) -> set[str]: +) -> AbstractSet[str]: owner_scopes: Final = get_resource_owner_scopes(user_api_key_dict) if not owner_scopes: return set() @@ -367,7 +401,7 @@ async def _get_allowed_container_ids( if prisma_client is None: return set() - rows: Final = await ManagedObjectRepository(prisma_client).table.find_many( + rows: Final[Sequence[_ManagedObjectRow]] = await ManagedObjectRepository(prisma_client).table.find_many( where={ "file_purpose": CONTAINER_OBJECT_PURPOSE, "created_by": {"in": owner_scopes}, @@ -382,10 +416,10 @@ async def _get_allowed_container_ids( async def filter_container_list_response( - response: Any, + response: _ContainerListResponse, user_api_key_dict: UserAPIKeyAuth, custom_llm_provider: str, -) -> Any: +) -> _ContainerListResponse: if is_proxy_admin(user_api_key_dict): return response @@ -394,7 +428,7 @@ async def filter_container_list_response( return response allowed_container_ids: Final = await _get_allowed_container_ids(user_api_key_dict) - filtered: Final[list[Any]] = [] + filtered: Final[list[object]] = [] for item in data: container_id = _get_response_id(item) if container_id is None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index c5481c9ce63..79867d492a1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -9,11 +9,13 @@ import contextlib import json import os import ssl -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping, Sequence +from ssl import SSLContext from typing import TYPE_CHECKING, Any, Final from fastapi import HTTPException from pydantic import BaseModel +from typing_extensions import NotRequired, TypedDict from websockets.asyncio.client import ClientConnection, connect from websockets.exceptions import ConnectionClosed @@ -35,8 +37,8 @@ from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( CallTypesLiteral, Choices, - EmbeddingResponse, - ImageResponse, + LLMResponseTypes, + Message, ModelResponse, ModelResponseStream, ResponsesAPIResponse, @@ -50,6 +52,40 @@ class CatoNetworksGuardrailMissingSecrets(Exception): pass +class _WsSslKwargs(TypedDict, total=False): + ssl: bool | str | SSLContext + + +class _CatoRequiredAction(TypedDict, total=False): + action_type: str + detection_message: str + + +class _CatoRedactedMessage(TypedDict): + role: NotRequired[str] + content: str | None + + +class _CatoRedactedChat(TypedDict, total=False): + all_redacted_messages: Sequence[_CatoRedactedMessage] + + +class _CatoAnalyzeResponse(TypedDict): + required_action: _CatoRequiredAction + analysis_result: NotRequired[Mapping[str, Mapping[str, object]]] + redacted_chat: NotRequired[_CatoRedactedChat] + + +class _CatoOutputRedaction(TypedDict): + redacted_output: str + + +class _CatoStreamMessage(TypedDict, total=False): + verified_chunk: Mapping[str, object] + done: bool + blocking_message: str + + class CatoNetworksGuardrail(CustomGuardrail): @classmethod def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: @@ -80,7 +116,7 @@ class CatoNetworksGuardrail(CustomGuardrail): super().__init__(**kwargs) @staticmethod - def _build_ws_ssl_kwargs(ssl_verify: bool | str | None, ws_api_base: str) -> dict: + def _build_ws_ssl_kwargs(ssl_verify: bool | str | None, ws_api_base: str) -> _WsSslKwargs: """Resolve the ``ssl`` argument for ``websockets.connect``. Mirrors the ``ssl_verify`` handling applied to the HTTP handler so a custom Cato instance behind TLS honours the same verification settings for streaming.""" @@ -156,7 +192,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return flattened @staticmethod - def _prompt_inspection_messages(prompt: Any) -> list: + def _prompt_inspection_messages(prompt: object) -> Sequence[Mapping[str, str]]: """Synthetic user messages for a legacy completion ``prompt`` (a string or a list of string prompts).""" if isinstance(prompt, str): @@ -208,7 +244,7 @@ class CatoNetworksGuardrail(CustomGuardrail): stack.extend(reversed(node)) @classmethod - def _extra_inspection_sources(cls, data: dict) -> list: + def _extra_inspection_sources(cls, data: dict) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: """Text the proxy forwards to the model outside chat ``messages``: Responses-API ``input`` and ``instructions``, legacy completion ``prompt`` and tool/function/``response_format`` schema strings. Returned @@ -251,7 +287,7 @@ class CatoNetworksGuardrail(CustomGuardrail): json={"messages": self._inspection_messages(data)}, ) response.raise_for_status() - res: Final = response.json() + res: Final[_CatoAnalyzeResponse] = response.json() required_action: Final = res.get("required_action") action_type: Final = required_action and required_action.get("action_type", None) if action_type is None: @@ -267,7 +303,11 @@ class CatoNetworksGuardrail(CustomGuardrail): verbose_proxy_logger.error("Cato: %s action", action_type) return data - def _handle_block_action(self, analysis_result: Any, required_action: Any) -> None: + def _handle_block_action( + self, + analysis_result: Mapping[str, Mapping[str, object]], + required_action: _CatoRequiredAction, + ) -> None: detection_message: Final = required_action.get("detection_message", None) verbose_proxy_logger.info( "Cato: Violation detected enabled policies: {policies}".format( @@ -348,7 +388,7 @@ class CatoNetworksGuardrail(CustomGuardrail): hook: str, key_alias: str | None, user_email: str | None = None, - ) -> dict | None: + ) -> _CatoOutputRedaction | None: call_id: Final = request_data.get("litellm_call_id") inspection_messages: Final = self._inspection_messages(request_data) assistant_index: Final = len(inspection_messages) @@ -363,7 +403,7 @@ class CatoNetworksGuardrail(CustomGuardrail): json={"messages": inspection_messages + [{"role": "assistant", "content": output}]}, ) response.raise_for_status() - res: Final = response.json() + res: Final[_CatoAnalyzeResponse] = response.json() required_action: Final = res.get("required_action") action_type: Final = required_action and required_action.get("action_type", None) if action_type and action_type == "block_action": @@ -378,7 +418,11 @@ class CatoNetworksGuardrail(CustomGuardrail): return {"redacted_output": redacted_output} return None - def _handle_block_action_on_output(self, analysis_result: Any, required_action: Any) -> None: + def _handle_block_action_on_output( + self, + analysis_result: Mapping[str, Mapping[str, object]], + required_action: _CatoRequiredAction, + ) -> None: detection_message: Final = required_action.get("detection_message", None) verbose_proxy_logger.info( "Cato: detected: {detected}, enabled policies: {policies}".format( @@ -422,7 +466,7 @@ class CatoNetworksGuardrail(CustomGuardrail): ) @staticmethod - def _output_fragments(message: Any) -> list: + def _output_fragments(message: Message) -> Sequence[tuple[tuple[str, int | None], str]]: """Assistant text the proxy returns to the caller: ``content`` plus every ``tool_calls[].function.arguments`` string, each tagged with where a redaction must be written back. ``content`` is only included when present @@ -439,7 +483,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return fragments @staticmethod - def _apply_output_fragment(message: Any, target: tuple, redacted: str) -> None: + def _apply_output_fragment(message: Any, target: tuple[str, int | None], redacted: str) -> None: kind, idx = target if kind == "content": message.content = redacted @@ -447,11 +491,11 @@ class CatoNetworksGuardrail(CustomGuardrail): message.tool_calls[idx].function.arguments = redacted @staticmethod - def _responses_output_field(item: Any, key: str) -> Any: + def _responses_output_field(item: object, key: str) -> str | Sequence[object] | None: return item.get(key) if isinstance(item, dict) else getattr(item, key, None) @classmethod - def _responses_output_fragments(cls, response: ResponsesAPIResponse) -> list: + def _responses_output_fragments(cls, response: ResponsesAPIResponse) -> Sequence[tuple[object, str, str]]: """Assistant text the Responses API returns to the caller: every ``output_text`` content block plus every function-call ``arguments`` string, each paired with the ``(container, key)`` a Cato redaction is @@ -474,7 +518,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return fragments @staticmethod - def _apply_responses_output_fragment(container: Any, key: str, redacted: str) -> None: + def _apply_responses_output_fragment(container: object, key: str, redacted: str) -> None: if isinstance(container, dict): container[key] = redacted else: @@ -505,8 +549,8 @@ class CatoNetworksGuardrail(CustomGuardrail): self, data: dict, user_api_key_dict: UserAPIKeyAuth, - response: Any | ModelResponse | EmbeddingResponse | ImageResponse, - ) -> Any: + response: LLMResponseTypes, + ) -> LLMResponseTypes: user_email: Final = self._resolve_cato_user_email(user_api_key_dict) if isinstance(response, ModelResponse) and response.choices: for choice in response.choices: @@ -526,7 +570,7 @@ class CatoNetworksGuardrail(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response, + response: AsyncGenerator[object, None], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: from litellm.proxy.proxy_server import StreamingCallbackError @@ -547,7 +591,7 @@ class CatoNetworksGuardrail(CustomGuardrail): try: while True: raw_message = await self._await_cato_message(websocket, sender) - result = json.loads(raw_message) + result: _CatoStreamMessage = json.loads(raw_message) if verified_chunk := result.get("verified_chunk"): yield ModelResponseStream.model_validate(verified_chunk) continue @@ -560,7 +604,7 @@ class CatoNetworksGuardrail(CustomGuardrail): finally: await self._cancel_background_task(sender) - async def _await_cato_message(self, websocket: ClientConnection, sender: asyncio.Task) -> Any: + async def _await_cato_message(self, websocket: ClientConnection, sender: asyncio.Task[None]) -> str | bytes: """Wait for the next Cato message, surfacing a dead forwarding task instead of blocking.""" from litellm.proxy.proxy_server import StreamingCallbackError @@ -578,7 +622,7 @@ class CatoNetworksGuardrail(CustomGuardrail): async def forward_the_stream_to_cato( self, websocket: ClientConnection, - response_iter: AsyncGenerator[Any, None], + response_iter: AsyncGenerator[object, None], ) -> None: async for chunk in response_iter: if isinstance(chunk, BaseModel): diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index 131ce5f9392..ca84ff47884 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -1,7 +1,8 @@ from __future__ import annotations import os -from typing import TYPE_CHECKING, Any, Final, Literal +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict from urllib.parse import urlparse from uuid import uuid4 @@ -29,9 +30,31 @@ from litellm.types.proxy.guardrails.guardrail_hooks.hiddenlayer import ( from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: + from pydantic import BaseModel + from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +class _HiddenlayerEvaluation(TypedDict, total=False): + action: str + threat_level: str + + +class _HiddenlayerAnalysisEntry(TypedDict, total=False): + name: str + detected: bool + + +class _HiddenlayerModifiedSide(TypedDict): + messages: Any + + +class _HiddenlayerResponse(TypedDict, total=False): + evaluation: _HiddenlayerEvaluation + analysis: Sequence[_HiddenlayerAnalysisEntry] + modified_data: Mapping[str, _HiddenlayerModifiedSide] + + def is_saas(host: str) -> bool: """Checks whether the connection is to the SaaS platform""" @@ -43,7 +66,7 @@ def is_saas(host: str) -> bool: return False -def _get_jwt(auth_url, api_id, api_key): +def _get_jwt(auth_url, api_id, api_key) -> str: token_url: Final = f"{auth_url}/oauth2/token?grant_type=client_credentials" resp: Final = requests.post(token_url, auth=HTTPBasicAuth(api_id, api_key)) @@ -139,7 +162,7 @@ class HiddenlayerGuardrail(CustomGuardrail): if scan_params := inputs.get("structured_messages"): last_msg: Final = scan_params[-1] - result = await self._call_hiddenlayer( + result: _HiddenlayerResponse = await self._call_hiddenlayer( project_id, hl_request_metadata, { @@ -205,11 +228,11 @@ class HiddenlayerGuardrail(CustomGuardrail): async def _call_hiddenlayer( self, project_id: str | None, - metadata: dict[str, str], - payload: dict[str, Any], + metadata: Mapping[str, str], + payload: Mapping[str, Sequence[Mapping[str, str]]], input_type: Literal["request", "response"], - ) -> dict[str, Any]: - data: Final[dict[str, Any]] = {"metadata": metadata} + ) -> _HiddenlayerResponse: + data: Final[dict[str, object]] = {"metadata": metadata} if input_type == "request": data["input"] = payload @@ -235,7 +258,7 @@ class HiddenlayerGuardrail(CustomGuardrail): headers=headers, ) response.raise_for_status() - result = response.json() + result: _HiddenlayerResponse = response.json() verbose_proxy_logger.debug("Hiddenlayer reponse: %s", result) @@ -265,7 +288,7 @@ class HiddenlayerGuardrail(CustomGuardrail): return result @staticmethod - def get_config_model() -> type[GuardrailConfigModel] | None: + def get_config_model() -> type[GuardrailConfigModel[BaseModel]] | None: from litellm.types.proxy.guardrails.guardrail_hooks.hiddenlayer import ( HiddenlayerGuardrailConfigModel, ) @@ -343,7 +366,7 @@ class HiddenlayerGuardrailV2(CustomGuardrail): if "hl-requester-id" not in hl_headers: hl_headers["hl-requester-id"] = "LiteLLM" - payload: Any + payload: object if input_type == "request": payload = { "messages": inputs.get("structured_messages"), @@ -461,7 +484,7 @@ class HiddenlayerGuardrailV2(CustomGuardrail): return response @staticmethod - def get_config_model() -> type[GuardrailConfigModel] | None: + def get_config_model() -> type[GuardrailConfigModel[BaseModel]] | None: from litellm.types.proxy.guardrails.guardrail_hooks.hiddenlayer import ( HiddenlayerGuardrailConfigModel, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py index 4ae29ade0d7..80beb90cf27 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py @@ -2,7 +2,10 @@ import threading import time import uuid from collections import OrderedDict -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from typing_extensions import NotRequired, TypedDict from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -15,6 +18,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import AllMessageValues GRAPH_API_BASE: Final = "https://graph.microsoft.com/v1.0" @@ -25,6 +29,11 @@ GRAPH_SCOPE: Final = "https://graph.microsoft.com/.default" SCOPE_CACHE_TTL_SECONDS: Final = 3600.0 +class GraphTokenResponse(TypedDict): + access_token: str + expires_in: NotRequired[int] + + class PurviewGuardrailBase: """ Base class for Microsoft Purview guardrails. @@ -41,8 +50,8 @@ class PurviewGuardrailBase: client_secret: str, purview_app_name: str = "LiteLLM", user_id_field: str = "user_id", - **kwargs: Any, - ): + **kwargs: object, + ) -> None: # Forward remaining kwargs to the next class in the MRO # (typically CustomGuardrail). super().__init__(**kwargs) @@ -59,7 +68,7 @@ class PurviewGuardrailBase: # Protection scope cache: user_id -> (etag, scope_response, fetched_at) # Capped at 1000 entries (LRU eviction) to avoid unbounded growth. - self._scope_cache: OrderedDict[str, tuple[str, dict[str, Any], float]] = OrderedDict() + self._scope_cache: OrderedDict[str, tuple[str, Mapping[str, object], float]] = OrderedDict() self._scope_cache_maxsize = 1000 # Use a threading.Lock (not asyncio.Lock) because this lock is acquired # from both the proxy's main asyncio event loop and from short-lived @@ -100,7 +109,7 @@ class PurviewGuardrailBase: headers={"Content-Type": "application/x-www-form-urlencoded"}, ) response.raise_for_status() - token_data: Final = response.json() + token_data: Final[GraphTokenResponse] = response.json() access_token: Final = token_data["access_token"] expires_in: Final = int(token_data.get("expires_in", 3599)) # Recompute ``now`` after the await so the expiry reflects when the @@ -117,9 +126,9 @@ class PurviewGuardrailBase: async def _graph_post( self, url: str, - json_body: dict[str, Any], - extra_headers: dict[str, str] | None = None, - ) -> tuple[dict[str, Any], dict[str, str]]: + json_body: dict[str, object], + extra_headers: Mapping[str, str] | None = None, + ) -> tuple[dict[str, object], dict[str, str]]: """POST to Graph API with bearer auth. Returns: @@ -136,7 +145,7 @@ class PurviewGuardrailBase: verbose_proxy_logger.debug("Purview Graph POST %s", url) response: Final = await self.async_handler.post(url=url, headers=headers, json=json_body) response.raise_for_status() - response_json: Final[dict[str, Any]] = response.json() + response_json: Final[dict[str, object]] = response.json() response_headers: Final = dict(response.headers) verbose_proxy_logger.debug("Purview Graph response: %s", response_json) return response_json, response_headers @@ -145,7 +154,7 @@ class PurviewGuardrailBase: # Protection scopes # ------------------------------------------------------------------ - async def _compute_protection_scopes(self, user_id: str) -> tuple[str, dict[str, Any]]: + async def _compute_protection_scopes(self, user_id: str) -> tuple[str, Mapping[str, object]]: """Call protectionScopes/compute and cache with ETag. Returns: @@ -161,7 +170,7 @@ class PurviewGuardrailBase: return cached[0], cached[1] url: Final = f"{GRAPH_API_BASE}/users/{encoded_user_id}/dataSecurityAndGovernance/protectionScopes/compute" - body: Final[dict[str, Any]] = { + body: Final[dict[str, object]] = { "activities": "uploadText,downloadText", "locations": [ { @@ -199,7 +208,7 @@ class PurviewGuardrailBase: activity: str, etag: str, correlation_id: str | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """Call processContent for DLP policy evaluation. Args: @@ -211,7 +220,7 @@ class PurviewGuardrailBase: """ encoded_user_id: Final = self._encode_graph_user_id(user_id) url: Final = f"{GRAPH_API_BASE}/users/{encoded_user_id}/dataSecurityAndGovernance/processContent" - body: Final[dict[str, Any]] = { + body: Final[dict[str, object]] = { "contentToProcess": { "contentEntries": [ { @@ -261,7 +270,9 @@ class PurviewGuardrailBase: # User ID resolution # ------------------------------------------------------------------ - def _resolve_user_id(self, data: dict[str, Any], user_api_key_dict: Any) -> str | None: + def _resolve_user_id( + self, data: Mapping[str, Mapping[str, object]], user_api_key_dict: "UserAPIKeyAuth" + ) -> str | None: """Resolve the Entra user object ID from request data or auth context. Returns the strongest available identity walking down four sources, in @@ -284,7 +295,7 @@ class PurviewGuardrailBase: if hasattr(user_api_key_dict, "end_user_id") and user_api_key_dict.end_user_id: return str(user_api_key_dict.end_user_id) - metadata: Final = data.get("metadata") or data.get("litellm_metadata") or {} + metadata: Final[Mapping[str, object]] = data.get("metadata") or data.get("litellm_metadata") or {} uid = metadata.get("user_api_key_user_id") if uid: return str(uid) @@ -296,15 +307,15 @@ class PurviewGuardrailBase: return None @staticmethod - def _logging_kwargs_metadata(kwargs: dict[str, Any]) -> dict[str, Any]: + def _logging_kwargs_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: """Metadata dict from ``model_call_details`` / logging kwargs.""" - litellm_params: Final = kwargs.get("litellm_params") or {} + litellm_params: Final[object] = kwargs.get("litellm_params") or {} if not isinstance(litellm_params, dict): return {} md: Final = litellm_params.get("metadata") return md if isinstance(md, dict) else {} - def _resolve_trusted_user_id(self, data: dict[str, Any], user_api_key_dict: Any) -> str | None: + def _resolve_trusted_user_id(self, data: Mapping[str, object], user_api_key_dict: "UserAPIKeyAuth") -> str | None: """Resolve user ID from API-key/JWT-bound identity for blocking DLP. Uses only ``UserAPIKeyAuth.user_id`` (bound on the LiteLLM key or JWT). @@ -325,7 +336,7 @@ class PurviewGuardrailBase: return None - def _resolve_user_id_from_logging_kwargs(self, kwargs: dict[str, Any]) -> str | None: + def _resolve_user_id_from_logging_kwargs(self, kwargs: Mapping[str, object]) -> str | None: """Trusted-identity-only resolver for logging-only hooks. Uses only the proxy-injected ``user_api_key_user_id`` (populated from @@ -348,7 +359,7 @@ class PurviewGuardrailBase: # ------------------------------------------------------------------ @staticmethod - def _should_block(response: dict[str, Any]) -> bool: + def _should_block(response: Mapping[str, Sequence[Mapping[str, str]]]) -> bool: """Return True if any policyAction requires blocking.""" for action in response.get("policyActions", []): odata_type = action.get("@odata.type", "") @@ -365,7 +376,7 @@ class PurviewGuardrailBase: # ------------------------------------------------------------------ @staticmethod - def is_token_id_prompt(prompt: Any) -> bool: + def is_token_id_prompt(prompt: str | Sequence[object] | None) -> bool: """Return True if ``prompt`` carries OpenAI completions token ids. Covers every list shape that ``completion_prompt_to_str`` cannot decode @@ -383,7 +394,7 @@ class PurviewGuardrailBase: return False @staticmethod - def completion_prompt_to_str(prompt: Any) -> str | None: + def completion_prompt_to_str(prompt: str | Sequence[object] | None) -> str | None: """Normalize OpenAI ``/v1/completions`` ``prompt`` for text DLP. Supports string prompts and list-of-string prompts. List-of-token-id prompts @@ -408,7 +419,7 @@ class PurviewGuardrailBase: return None @staticmethod - def _extract_tool_call_args_from_message(message: Any) -> list[str]: + def _extract_tool_call_args_from_message(message: object) -> list[str]: """Return plaintext arguments strings from tool_calls and function_call fields. Covers both the request path (assistant messages in chat histories that @@ -419,7 +430,9 @@ class PurviewGuardrailBase: args: Final[list[str]] = [] # tool_calls: [{"function": {"arguments": "..."}}] - tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) + tool_calls: Final[Sequence[object] | None] = ( + message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) + ) if tool_calls: for tc in tool_calls: fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) diff --git a/litellm/proxy/management_endpoints/workflow_management_endpoints.py b/litellm/proxy/management_endpoints/workflow_management_endpoints.py index 70a6cc507f5..eeb64ab2773 100644 --- a/litellm/proxy/management_endpoints/workflow_management_endpoints.py +++ b/litellm/proxy/management_endpoints/workflow_management_endpoints.py @@ -14,7 +14,8 @@ GET /v1/workflows/runs/{run_id}/messages - Fetch conversation history """ import json -from typing import Any, Final, Literal +from collections.abc import Mapping, Sequence +from typing import Final, Literal, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query @@ -43,7 +44,7 @@ router: Final = APIRouter() _MAX_SEQUENCE_RETRIES: Final = 5 -def _json(value: Any) -> str: +def _json(value: object) -> str: """Serialize a Python value for prisma-client-py Json fields (must be a string).""" return json.dumps(value) @@ -62,7 +63,7 @@ def _caller_key(user_api_key_dict: UserAPIKeyAuth) -> str | None: # Status transitions driven by event_type -_EVENT_STATUS_MAP: Final[dict[str, str]] = { +_EVENT_STATUS_MAP: Final[Mapping[str, str]] = { "step.started": "running", "step.failed": "failed", "hook.waiting": "paused", @@ -77,8 +78,8 @@ _EVENT_STATUS_MAP: Final[dict[str, str]] = { class WorkflowRunCreateRequest(BaseModel): workflow_type: str - input: dict[str, Any] | None = None - metadata: dict[str, Any] | None = None + input: Mapping[str, object] | None = None + metadata: Mapping[str, object] | None = None WorkflowRunStatus = Literal["pending", "running", "paused", "completed", "failed"] @@ -86,14 +87,14 @@ WorkflowRunStatus = Literal["pending", "running", "paused", "completed", "failed class WorkflowRunUpdateRequest(BaseModel): status: WorkflowRunStatus | None = None - output: dict[str, Any] | None = None - metadata: dict[str, Any] | None = None + output: Mapping[str, object] | None = None + metadata: Mapping[str, object] | None = None class WorkflowEventCreateRequest(BaseModel): event_type: str step_name: str - data: dict[str, Any] | None = None + data: Mapping[str, object] | None = None class WorkflowMessageCreateRequest(BaseModel): @@ -102,15 +103,60 @@ class WorkflowMessageCreateRequest(BaseModel): session_id: str | None = None +class _RunRow(Protocol): + @property + def created_by(self) -> str | None: ... + + +class _SeqRow(Protocol): + @property + def sequence_number(self) -> int: ... + + +class _RunCreateData(TypedDict, total=False): + workflow_type: str + created_by: str | None + input: str + metadata: str + + +class _RunWhere(TypedDict, total=False): + workflow_type: str + status: str | Mapping[str, Sequence[str]] + created_by: str + + +class _RunUpdateData(TypedDict, total=False): + status: WorkflowRunStatus + output: str + metadata: str + + +class _EventCreateData(TypedDict, total=False): + run_id: str + event_type: str + step_name: str + sequence_number: int + data: str + + +class _MessageCreateData(TypedDict, total=False): + run_id: str + role: str + content: str + sequence_number: int + session_id: str + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -async def _get_next_sequence_number(prisma_client: Any, run_id: str, table: str) -> int: +async def _get_next_sequence_number(prisma_client: object, run_id: str, table: str) -> int: """Return MAX(sequence_number) + 1 for the given run, for either events or messages.""" if table == "events": - rows = await WorkflowEventRepository(prisma_client).table.find_many( + rows: Sequence[_SeqRow] = await WorkflowEventRepository(prisma_client).table.find_many( where={"run_id": run_id}, order={"sequence_number": "desc"}, take=1, @@ -125,12 +171,12 @@ async def _get_next_sequence_number(prisma_client: Any, run_id: str, table: str) async def _require_run( - prisma_client: Any, + prisma_client: object, run_id: str, user_api_key_dict: UserAPIKeyAuth | None = None, -) -> Any: +) -> _RunRow: """Return the run or raise 404. For non-admin callers, also enforce key ownership.""" - run: Final = await WorkflowRunRepository(prisma_client).table.find_unique(where={"run_id": run_id}) + run: Final[_RunRow | None] = await WorkflowRunRepository(prisma_client).table.find_unique(where={"run_id": run_id}) if run is None: raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found") if user_api_key_dict is not None and not _is_admin(user_api_key_dict): @@ -165,7 +211,7 @@ async def create_workflow_run( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - create_data: Final[dict[str, Any]] = { + create_data: Final[_RunCreateData] = { "workflow_type": data.workflow_type, "created_by": _caller_key(user_api_key_dict), } @@ -173,7 +219,7 @@ async def create_workflow_run( create_data["input"] = _json(data.input) if data.metadata is not None: create_data["metadata"] = _json(data.metadata) - run: Final = await WorkflowRunRepository(prisma_client).table.create(data=create_data) + run: Final[_RunRow] = await WorkflowRunRepository(prisma_client).table.create(data=create_data) return run except Exception as e: verbose_proxy_logger.exception("Error creating workflow run: %s", e) @@ -200,7 +246,7 @@ async def list_workflow_runs( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - where: Final[dict[str, Any]] = {} + where: Final[_RunWhere] = {} if workflow_type: where["workflow_type"] = workflow_type if status: @@ -214,7 +260,7 @@ async def list_workflow_runs( where["created_by"] = caller try: - runs: Final = await WorkflowRunRepository(prisma_client).table.find_many( + runs: Final[Sequence[object]] = await WorkflowRunRepository(prisma_client).table.find_many( where=where, order={"created_at": "desc"}, take=limit, @@ -241,7 +287,7 @@ async def get_workflow_run( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - run: Final = await WorkflowRunRepository(prisma_client).table.find_unique( + run: Final[_RunRow | None] = await WorkflowRunRepository(prisma_client).table.find_unique( where={"run_id": run_id}, include={"events": {"order_by": {"sequence_number": "desc"}, "take": 1}}, ) @@ -275,7 +321,7 @@ async def update_workflow_run( if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - update: Final[dict[str, Any]] = {} + update: Final[_RunUpdateData] = {} if data.status is not None: update["status"] = data.status if data.output is not None: @@ -290,7 +336,7 @@ async def update_workflow_run( await _require_run(prisma_client, run_id, user_api_key_dict) try: - run: Final = await WorkflowRunRepository(prisma_client).table.update( + run: Final[_RunRow | None] = await WorkflowRunRepository(prisma_client).table.update( where={"run_id": run_id}, data=update, ) @@ -332,7 +378,7 @@ async def append_workflow_event( for attempt in range(_MAX_SEQUENCE_RETRIES): try: seq = await _get_next_sequence_number(prisma_client, run_id, "events") - event_data: dict[str, Any] = { + event_data: _EventCreateData = { "run_id": run_id, "event_type": data.event_type, "step_name": data.step_name, @@ -342,7 +388,7 @@ async def append_workflow_event( event_data["data"] = _json(data.data) async with prisma_client.db.tx() as tx: - event = await tx.litellm_workflowevent.create(data=event_data) + event: object = await tx.litellm_workflowevent.create(data=event_data) if new_status: await tx.litellm_workflowrun.update( where={"run_id": run_id}, @@ -389,7 +435,7 @@ async def list_workflow_events( await _require_run(prisma_client, run_id, _read_scope_caller(user_api_key_dict)) try: - events: Final = await WorkflowEventRepository(prisma_client).table.find_many( + events: Final[Sequence[object]] = await WorkflowEventRepository(prisma_client).table.find_many( where={"run_id": run_id}, order={"sequence_number": "asc"}, take=limit, @@ -424,7 +470,7 @@ async def append_workflow_message( for attempt in range(_MAX_SEQUENCE_RETRIES): try: seq = await _get_next_sequence_number(prisma_client, run_id, "messages") - msg_data: dict[str, Any] = { + msg_data: _MessageCreateData = { "run_id": run_id, "role": data.role, "content": data.content, @@ -432,7 +478,7 @@ async def append_workflow_message( } if data.session_id is not None: msg_data["session_id"] = data.session_id - msg = await WorkflowMessageRepository(prisma_client).table.create(data=msg_data) + msg: object = await WorkflowMessageRepository(prisma_client).table.create(data=msg_data) return msg except Exception as e: @@ -473,7 +519,7 @@ async def list_workflow_messages( await _require_run(prisma_client, run_id, _read_scope_caller(user_api_key_dict)) try: - messages: Final = await WorkflowMessageRepository(prisma_client).table.find_many( + messages: Final[Sequence[object]] = await WorkflowMessageRepository(prisma_client).table.find_many( where={"run_id": run_id}, order={"sequence_number": "asc"}, take=limit, diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py index 8ee1dd268e6..05df44242aa 100644 --- a/litellm/proxy/policy_engine/attachment_registry.py +++ b/litellm/proxy/policy_engine/attachment_registry.py @@ -6,7 +6,7 @@ This allows the same policy to be attached to multiple scopes. """ from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypedDict from litellm._logging import verbose_proxy_logger from litellm.repositories.table_repositories import PolicyAttachmentRepository @@ -18,9 +18,18 @@ from litellm.types.proxy.policy_engine import ( ) if TYPE_CHECKING: + from collections.abc import Sequence + + from prisma.models import LiteLLM_PolicyAttachmentTable + from litellm.proxy.utils import PrismaClient +class PolicyAttachmentMatch(TypedDict): + policy_name: str + matched_via: str + + class AttachmentRegistry: """ In-memory registry for storing and managing policy attachments. @@ -40,7 +49,7 @@ class AttachmentRegistry: ``` """ - def __init__(self): + def __init__(self) -> None: self._attachments: list[PolicyAttachment] = [] self._config_attachments: tuple[PolicyAttachment, ...] = () self._initialized: bool = False @@ -98,7 +107,7 @@ class AttachmentRegistry: """ return [r["policy_name"] for r in self.get_attached_policies_with_reasons(context)] - def get_attached_policies_with_reasons(self, context: PolicyMatchContext) -> list[dict[str, Any]]: + def get_attached_policies_with_reasons(self, context: PolicyMatchContext) -> list[PolicyAttachmentMatch]: """ Get list of policy names and match reasons for the given context. @@ -107,8 +116,8 @@ class AttachmentRegistry: """ from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher - results: Final[list[dict[str, Any]]] = [] - seen_policies: Final[set] = set() + results: Final[list[PolicyAttachmentMatch]] = [] + seen_policies: Final[set[str]] = set() for attachment in self._attachments: scope = attachment.to_policy_scope() @@ -280,7 +289,9 @@ class AttachmentRegistry: PolicyAttachmentDBResponse with the created attachment """ try: - created_attachment: Final = await PolicyAttachmentRepository(prisma_client).table.create( + created_attachment: Final[LiteLLM_PolicyAttachmentTable] = await PolicyAttachmentRepository( + prisma_client + ).table.create( data={ "policy_name": attachment_request.policy_name, "scope": attachment_request.scope, @@ -340,9 +351,9 @@ class AttachmentRegistry: """ try: # Get attachment before deleting - attachment: Final = await PolicyAttachmentRepository(prisma_client).table.find_unique( - where={"attachment_id": attachment_id} - ) + attachment: Final[LiteLLM_PolicyAttachmentTable | None] = await PolicyAttachmentRepository( + prisma_client + ).table.find_unique(where={"attachment_id": attachment_id}) if attachment is None: raise Exception(f"Attachment with ID {attachment_id} not found") @@ -375,9 +386,9 @@ class AttachmentRegistry: PolicyAttachmentDBResponse if found, None otherwise """ try: - attachment: Final = await PolicyAttachmentRepository(prisma_client).table.find_unique( - where={"attachment_id": attachment_id} - ) + attachment: Final[LiteLLM_PolicyAttachmentTable | None] = await PolicyAttachmentRepository( + prisma_client + ).table.find_unique(where={"attachment_id": attachment_id}) if attachment is None: return None @@ -413,7 +424,9 @@ class AttachmentRegistry: List of PolicyAttachmentDBResponse objects """ try: - attachments: Final = await PolicyAttachmentRepository(prisma_client).table.find_many( + attachments: Final[Sequence[LiteLLM_PolicyAttachmentTable]] = await PolicyAttachmentRepository( + prisma_client + ).table.find_many( order={"created_at": "desc"}, ) From 5339ec50e788a6bb9090380d1e43060e13bcdb97 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:42:20 -0700 Subject: [PATCH 047/672] fix(batches): persist managed file ids for cancelled/failed/expired batches When the batch cost poller found a batch in a terminal failed, expired, or cancelled state it wrote the provider response straight to the managed object table, so the stored blob kept raw provider file ids and a raw batch id. Since the row is final after batch_processed=True and the read paths only resolve existing managed ids, every later GET /batches/{id} and GET /batches leaked raw provider output and error file ids that clients cannot fetch through the proxy. The terminal branch now normalizes the response with ensure_batch_response_managed_file_ids before persisting, minting managed ids under the batch owner's identity POST /batches/{id}/cancel had the same gap: it called update_batch_in_database without the caller's auth context, so a cancel response that already carried provider file ids could never mint managed ids. The endpoint now forwards user_api_key_dict --- .../proxy/common_utils/check_batch_cost.py | 14 ++ litellm/proxy/batches_endpoints/endpoints.py | 1 + .../proxy_unit_tests/test_check_batch_cost.py | 151 +++++++++++++++--- .../proxy/batches_endpoints/test_endpoints.py | 11 ++ 4 files changed, 158 insertions(+), 19 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 3ed63b0d9ee..a12b0bb7170 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -656,6 +656,20 @@ class CheckBatchCost: elif response.status in ("failed", "expired", "cancelled"): try: + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + ensure_batch_response_managed_file_ids, + ) + + response.id = job.unified_object_id + await ensure_batch_response_managed_file_ids( + response=response, + managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"), + prisma_client=self.prisma_client, + verbose_proxy_logger=verbose_proxy_logger, + db_batch_object=job, + unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id), + ) update_data = { "status": response.status, "file_object": response.model_dump_json(), diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index deed665d3f2..7b5af6c1068 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -961,6 +961,7 @@ async def cancel_batch( prisma_client=prisma_client, verbose_proxy_logger=verbose_proxy_logger, operation="cancel", + user_api_key_dict=user_api_key_dict, ) ### CALL HOOKS ### - modify outgoing data diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index a15abd023d8..d6c5d8fc809 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -499,7 +499,7 @@ class TestCheckBatchCost: must be written back with that status and batch_processed=True so it stops being polled forever. """ - from unittest.mock import patch + import base64 mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( return_value=0 @@ -511,7 +511,9 @@ class TestCheckBatchCost: mock_job = MagicMock() mock_job.id = "job-terminal-1" - mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA==" + mock_job.unified_object_id = base64.urlsafe_b64encode( + b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456" + ).decode() mock_job.created_by = "user-1" assert check_batch_cost_instance._has_batch_processed_column is True @@ -527,23 +529,7 @@ class TestCheckBatchCost: mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) - decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;" - - with ( - patch( - "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id", - side_effect=[decoded_id, None], - ), - patch( - "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id", - return_value="model-123", - ), - patch( - "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id", - return_value="batch-456", - ), - ): - await check_batch_cost_instance.check_batch_cost() + await check_batch_cost_instance.check_batch_cost() assert ( mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 @@ -556,6 +542,133 @@ class TestCheckBatchCost: update_data["batch_processed"] is True ), "terminal-status update() must set batch_processed=True so polling stops" + @pytest.mark.asyncio + @pytest.mark.parametrize("terminal_status", ["failed", "expired", "cancelled"]) + async def test_terminal_status_persists_managed_output_file_ids( + self, + check_batch_cost_instance, + mock_prisma_client, + mock_llm_router, + terminal_status, + ): + """A cancelled/failed/expired batch with provider output files must be persisted + with unified managed file IDs, never raw provider IDs. Raw IDs written here leak + to every later GET /batches/{id} and GET /batches because the terminal row is + final (batch_processed=True) and read paths only resolve, never mint. + """ + import base64 + import json + + from litellm.types.utils import LiteLLMBatch + + unified_batch_uid = base64.urlsafe_b64encode( + b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456" + ).decode() + raw_output_file_id = "file-terminal-out-abc" + raw_error_file_id = "file-terminal-err-xyz" + raw_input_file_id = "file-terminal-in-123" + unified_input_file_id = base64.urlsafe_b64encode( + b"litellm_proxy:application/octet-stream;unified_id,in-1;target_model_names,gpt-5-batch" + ).decode() + unified_output_file_id = base64.urlsafe_b64encode( + f"litellm_proxy:application/octet-stream;unified_id,u-1;llm_output_file_id,{raw_output_file_id}".encode() + ).decode() + unified_error_file_id = base64.urlsafe_b64encode( + f"litellm_proxy:application/octet-stream;unified_id,u-2;llm_output_file_id,{raw_error_file_id}".encode() + ).decode() + + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=None + ) + + input_file_row = MagicMock() + input_file_row.unified_file_id = unified_input_file_id + + def find_managed_file(where): + if where["flat_model_file_ids"]["has"] == raw_input_file_id: + return input_file_row + return None + + mock_prisma_client.db.litellm_managedfiletable.find_first = AsyncMock( + side_effect=find_managed_file + ) + + mock_job = MagicMock() + mock_job.id = "job-terminal-mint-1" + mock_job.unified_object_id = unified_batch_uid + mock_job.created_by = "user-1" + mock_job.team_id = "team-1" + + check_batch_cost_instance._has_batch_processed_column = True + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[mock_job] + ) + + response = LiteLLMBatch( + id="batch-456", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id=raw_input_file_id, + object="batch", + status=terminal_status, + output_file_id=raw_output_file_id, + error_file_id=raw_error_file_id, + ) + mock_llm_router.aretrieve_batch = AsyncMock(return_value=response) + + mock_hook = MagicMock() + mock_hook.get_unified_output_file_id.side_effect = [ + unified_output_file_id, + unified_error_file_id, + ] + mock_hook.store_unified_file_id = AsyncMock() + check_batch_cost_instance.proxy_logging_obj.get_proxy_hook.return_value = ( + mock_hook + ) + + await check_batch_cost_instance.check_batch_cost() + + mock_hook.get_unified_output_file_id.assert_any_call( + output_file_id=raw_output_file_id, + model_id="model-123", + model_name="gpt-5-batch", + ) + mock_hook.get_unified_output_file_id.assert_any_call( + output_file_id=raw_error_file_id, + model_id="model-123", + model_name="gpt-5-batch", + ) + stored = { + next(iter(c.kwargs["model_mappings"].values())): c.kwargs["file_id"] + for c in mock_hook.store_unified_file_id.call_args_list + } + assert stored == { + raw_output_file_id: unified_output_file_id, + raw_error_file_id: unified_error_file_id, + } + for store_call in mock_hook.store_unified_file_id.call_args_list: + assert store_call.kwargs["user_api_key_dict"].user_id == "user-1" + assert store_call.kwargs["user_api_key_dict"].team_id == "team-1" + + assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 + update_call = mock_prisma_client.db.litellm_managedobjecttable.update.call_args + assert update_call.kwargs["where"] == {"id": "job-terminal-mint-1"} + update_data = update_call.kwargs["data"] + assert update_data["status"] == terminal_status + assert update_data["batch_processed"] is True + persisted = json.loads(update_data["file_object"]) + assert persisted["id"] == unified_batch_uid + assert persisted["input_file_id"] == unified_input_file_id + assert persisted["output_file_id"] == unified_output_file_id + assert persisted["error_file_id"] == unified_error_file_id + assert raw_output_file_id not in update_data["file_object"] + assert raw_error_file_id not in update_data["file_object"] + @pytest.mark.asyncio async def test_raw_output_file_id_converted_to_managed_id( self, check_batch_cost_instance, mock_prisma_client, mock_llm_router diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index f2d37fbe842..64e1dcda5c9 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -1896,6 +1896,17 @@ async def test_cancel__unified_batch_id_routes_to_router(cancel_harness): assert cancel_harness.update_batch_in_db.call_args.kwargs["operation"] == "cancel" +@pytest.mark.asyncio +async def test_cancel__db_write_receives_caller_auth(cancel_harness): + """update_batch_in_database can only mint managed IDs for a cancelled batch's + output files when it has an auth context, so cancel must forward the caller's.""" + caller = UserAPIKeyAuth(api_key="sk-test", user_id="user-cancel-1") + with patch.object(endpoints, "_is_base64_encoded_unified_file_id", return_value=UNIFIED_BATCH_ID): + await call_cancel(cancel_harness, "batch-unified-blob", user=caller) + + assert cancel_harness.update_batch_in_db.call_args.kwargs["user_api_key_dict"] is caller + + @pytest.mark.asyncio async def test_cancel__unified_missing_model_id_400(cancel_harness): # unified id with no model_id segment -> get_model_id returns None -> 400. From 0991692e684fb00601c9ad7fe0b7d9dbddb602e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:44:25 -0700 Subject: [PATCH 048/672] test: roll back runtime model registrations between tests Since #35491, register_model records every registration in the process-global _runtime_registered_model_cost ledger, and every cost map swap replays that ledger on top of the freshly adopted map. Under pytest-xdist, any earlier test in the same worker that registered gpt-3.5-turbo leaked into TestPriceDataReloadIntegration::test_distributed_reload_check_function: the replay ballooned its sparse mocked entry into a full ModelInfo dict and failed the exact-equality assert, breaking the proxy-infra shard whenever loadscope happened to co-schedule such a test first (reruns cannot help since the pollution is process-wide) The autouse isolate_litellm_state fixture now snapshots the ledger before each test and restores it in place on teardown, so no test's registrations outlive it. A regression pair in test_conftest_isolation.py asserts the rollback --- tests/test_litellm/conftest.py | 9 +++++++++ tests/test_litellm/test_conftest_isolation.py | 13 +++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/test_litellm/test_conftest_isolation.py diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index f4aa1926d21..c0993051d33 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -19,6 +19,7 @@ sys.path.insert( import asyncio import litellm +from litellm import utils as litellm_utils_module from litellm._logging import ALL_LOGGERS from litellm.litellm_core_utils.prompt_templates import ( image_handling as image_handling_module, @@ -238,6 +239,11 @@ def isolate_litellm_state(): if hasattr(litellm, _attr): original_state[_attr] = getattr(litellm, _attr) + original_runtime_registered_model_cost = { + model_key: dict(model_value) + for model_key, model_value in litellm_utils_module._runtime_registered_model_cost.items() + } + # Store LiteLLM logger state. Some tests reconfigure handlers/propagation for # JSON logging and do not restore them, which breaks later caplog-based tests. logger_state = {} @@ -304,6 +310,9 @@ def isolate_litellm_state(): if hasattr(litellm, attr_name): setattr(litellm, attr_name, original_value) + litellm_utils_module._runtime_registered_model_cost.clear() + litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost) + # Restore logger configuration mutated by logging-focused tests. for logger in ALL_LOGGERS: original_logger_state = logger_state.get(logger.name) diff --git a/tests/test_litellm/test_conftest_isolation.py b/tests/test_litellm/test_conftest_isolation.py new file mode 100644 index 00000000000..88889ad7740 --- /dev/null +++ b/tests/test_litellm/test_conftest_isolation.py @@ -0,0 +1,13 @@ +import litellm +from litellm import utils as litellm_utils_module + +CANARY_MODEL = "conftest-isolation-canary-model" + + +def test_register_model_ledger_entry_is_scoped_to_this_test(): + litellm.register_model({CANARY_MODEL: {"litellm_provider": "openai", "input_cost_per_token": 0.001}}) + assert CANARY_MODEL in litellm_utils_module._runtime_registered_model_cost + + +def test_register_model_ledger_entry_was_rolled_back(): + assert CANARY_MODEL not in litellm_utils_module._runtime_registered_model_cost From 7d00f9d019f84be709a7515094fed4ce7bbee900 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:46:00 -0700 Subject: [PATCH 049/672] fix(managed_files): return unified output file ids from GET /batches list_user_batches parsed each stored batch blob and returned it as-is, so any row whose blob still carried raw provider file ids (for example a batch that reached a terminal state through the cost poller, or rows written before output registration existed) leaked raw output_file_id and error_file_id values that clients cannot fetch through the proxy. The list path now runs each row through ensure_batch_response_managed_file_ids, which swaps in existing managed ids and registers missing ones under the batch owner's identity, matching what GET /batches/{id} already does --- .../proxy/hooks/managed_files.py | 12 ++ .../proxy/hooks/test_managed_files.py | 142 ++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 0036603bcd1..07a1f959940 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -31,6 +31,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + ensure_batch_response_managed_file_ids, get_batch_id_from_unified_batch_id, get_content_type_from_file_object, get_model_id_from_unified_batch_id, @@ -352,6 +353,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) batch_obj = LiteLLMBatch.model_validate(batch_data) batch_obj.id = batch.unified_object_id + await ensure_batch_response_managed_file_ids( + response=batch_obj, + managed_files_obj=self, + prisma_client=self.prisma_client, + verbose_proxy_logger=verbose_logger, + user_api_key_dict=user_api_key_dict, + db_batch_object=batch, + unified_batch_id=_is_base64_encoded_unified_file_id( + batch.unified_object_id + ), + ) batch_objects.append(batch_obj) except Exception as e: diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 50af6465d06..fc10a1257e1 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -1813,6 +1813,148 @@ def _create_unified_batch_id(model_id: str, batch_id: str) -> str: return base64.urlsafe_b64encode(unified_str.encode()).decode().rstrip("=") +def _decode_unified_id(b64_id: str) -> str: + return base64.urlsafe_b64decode(b64_id + "=" * (-len(b64_id) % 4)).decode() + + +def _terminal_batch_record( + unified_batch_uid: str, + raw_input_file_id: str, + raw_output_file_id: str, + raw_error_file_id: str, +): + record = MagicMock() + record.unified_object_id = unified_batch_uid + record.created_by = "owner-user" + record.team_id = "owner-team" + record.status = "cancelled" + record.file_object = json.dumps( + { + "id": "batch-raw-456", + "object": "batch", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "status": "cancelled", + "created_at": 1234567890, + "input_file_id": raw_input_file_id, + "output_file_id": raw_output_file_id, + "error_file_id": raw_error_file_id, + } + ) + return record + + +@pytest.mark.asyncio +async def test_list_batches_registers_and_returns_unified_output_file_ids(): + """A stored batch blob with raw provider file IDs (e.g. persisted by the cost + poller for a cancelled batch) must be listed with unified managed IDs, and the + output/error files must be registered in the managed file table so GET + /files/{id}/content can route them.""" + from litellm.proxy._types import UserAPIKeyAuth + + unified_batch_uid = _create_unified_batch_id("model-123", "batch-456") + raw_input_file_id = "file-list-in-1" + raw_output_file_id = "file-list-out-1" + raw_error_file_id = "file-list-err-1" + unified_input_file_id = base64.urlsafe_b64encode( + b"litellm_proxy:application/octet-stream;unified_id,in-1;target_model_names,gpt-5-batch" + ).decode() + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_many.return_value = [ + _terminal_batch_record( + unified_batch_uid, raw_input_file_id, raw_output_file_id, raw_error_file_id + ) + ] + + input_file_row = MagicMock() + input_file_row.unified_file_id = unified_input_file_id + + def find_managed_file(where): + if where["flat_model_file_ids"]["has"] == raw_input_file_id: + return input_file_row + return None + + prisma_client.db.litellm_managedfiletable.find_first = AsyncMock( + side_effect=find_managed_file + ) + + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.list_user_batches( + user_api_key_dict=UserAPIKeyAuth(user_id="owner-user"), + limit=10, + ) + + listed = result["data"][0] + assert listed.id == unified_batch_uid + assert listed.input_file_id == unified_input_file_id + + decoded_output = _decode_unified_id(listed.output_file_id) + assert decoded_output.startswith("litellm_proxy") + assert f"llm_output_file_id,{raw_output_file_id}" in decoded_output + assert "llm_output_file_model_id,model-123" in decoded_output + assert "target_model_names,gpt-5-batch" in decoded_output + + decoded_error = _decode_unified_id(listed.error_file_id) + assert f"llm_output_file_id,{raw_error_file_id}" in decoded_error + + upsert_calls = prisma_client.db.litellm_managedfiletable.upsert.await_args_list + stored_raw_ids = { + c.kwargs["data"]["create"]["flat_model_file_ids"][0] for c in upsert_calls + } + assert stored_raw_ids == {raw_output_file_id, raw_error_file_id} + for c in upsert_calls: + assert c.kwargs["data"]["create"]["created_by"] == "owner-user" + assert c.kwargs["data"]["create"]["team_id"] == "owner-team" + + +@pytest.mark.asyncio +async def test_list_batches_resolves_existing_managed_rows_without_minting(): + """When the raw provider file IDs already have managed file rows, listing must + swap in the existing unified IDs and must not upsert duplicate rows.""" + from litellm.proxy._types import UserAPIKeyAuth + + unified_batch_uid = _create_unified_batch_id("model-123", "batch-456") + raw_output_file_id = "file-list-out-existing" + existing_unified_output_id = base64.urlsafe_b64encode( + f"litellm_proxy:application/json;unified_id,u-9;llm_output_file_id,{raw_output_file_id}".encode() + ).decode() + + record = _terminal_batch_record( + unified_batch_uid, "file-list-in-9", raw_output_file_id, "" + ) + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_many.return_value = [record] + + existing_row = MagicMock() + existing_row.unified_file_id = existing_unified_output_id + + def find_managed_file(where): + if where["flat_model_file_ids"]["has"] == raw_output_file_id: + return existing_row + return None + + prisma_client.db.litellm_managedfiletable.find_first = AsyncMock( + side_effect=find_managed_file + ) + + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.list_user_batches( + user_api_key_dict=UserAPIKeyAuth(user_id="owner-user"), + limit=10, + ) + + assert result["data"][0].output_file_id == existing_unified_output_id + prisma_client.db.litellm_managedfiletable.upsert.assert_not_awaited() + + @pytest.mark.asyncio async def test_list_batches_from_managed_objects_table_provider_filter_raises_exception(): from litellm.proxy._types import UserAPIKeyAuth From c2998dea7510a3b656c06d54dbcdae769927b83d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:49:15 -0700 Subject: [PATCH 050/672] fix(guardrails): guard tools write-back under scan_only_tool_results and warn on role-filtered no-op scans --- .../chat/guardrail_translation/handler.py | 2 +- .../chat/guardrail_translation/handler.py | 2 +- .../guardrail_hooks/bedrock_guardrails.py | 8 +++ .../panw_prisma_airs/panw_prisma_airs.py | 12 +++++ .../test_anthropic_guardrail_handler.py | 34 ++++++++++++ .../test_openai_guardrail_handler.py | 54 +++++++++++++++++++ .../test_bedrock_guardrails.py | 37 +++++++++++++ .../guardrail_hooks/test_panw_prisma_airs.py | 28 ++++++++++ 8 files changed, 175 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 60424fb78b5..184e0f6a343 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -387,7 +387,7 @@ class AnthropicMessagesHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) guardrailed_tools: Final = guardrailed_inputs.get("tools") - if guardrailed_tools is not None: + if guardrailed_tools is not None and not scan_only_tool_results: # Convert tools back from OpenAI format to Anthropic format anthropic_config: Final = AnthropicConfig() anthropic_tools: Final[list[AllAnthropicToolsValues]] = [] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 9d7fe6ce2a8..dc2a06d67fc 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -143,7 +143,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", []) guardrailed_tools: Final = guardrailed_inputs.get("tools") - if guardrailed_tools is not None: + if guardrailed_tools is not None and not scan_only_tool_results: data["tools"] = guardrailed_tools guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages") diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index f7a5c7559b1..8193069fd82 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -26,6 +26,9 @@ from litellm.caching import DualCache from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys +from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_scan_only_tool_results_for_guardrail, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -523,6 +526,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): latest_user_index: Final = self._find_latest_message_index(structured_messages, target_role="user") if latest_user_index is None: + if effective_scan_only_tool_results_for_guardrail(self): + verbose_proxy_logger.warning( + "Bedrock Guardrail: experimental_use_latest_role_message_only scans only the latest " + "user message, so scan_only_tool_results leaves nothing to scan for this request" + ) verbose_proxy_logger.debug("Bedrock Guardrail: no user-role message in request, skipping INPUT scan") return ApplyGuardrailMessageSelection(None, None, True, skip_scan=True) diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index ae1478a9210..a96a0070eef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -22,6 +22,9 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_scan_only_tool_results_for_guardrail, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -1716,6 +1719,15 @@ class PanwPrismaAirsHandler(CustomGuardrail): # - latest-user extraction returned None (no user / count mismatch) if scannable_indices is None: scannable_indices = self._get_scannable_text_indices(texts, structured_messages) + if ( + scannable_indices is not None + and not scannable_indices + and effective_scan_only_tool_results_for_guardrail(self) + ): + verbose_proxy_logger.warning( + "PANW Prisma AIRS scans only user, system, and developer messages, " + "so scan_only_tool_results leaves nothing to scan for this request" + ) for i, text in enumerate(texts): if not text or not text.strip(): diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index a016e1a2deb..c7dedff0663 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -882,6 +882,40 @@ class TestAnthropicMessagesScanOnlyToolResults: ) assert data["messages"][2]["content"][0]["text"] == "sibling POISON text" + @pytest.mark.asyncio + async def test_guardrail_synthesized_tools_never_replace_scoped_out_request_tools(self): + handler = AnthropicMessagesHandler() + guardrail = ToolAppendingGuardrail(guardrail_name="tool-appending") + guardrail.scan_only_tool_results = True + original_tools = [ + { + "name": "get_weather", + "description": "Get the weather at a specific location", + "input_schema": {"type": "object", "properties": {"location": {"type": "string"}}}, + } + ] + data = { + "model": "claude-sonnet-4-5", + "tools": original_tools, + "messages": [ + {"role": "user", "content": "what's the weather?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu1", "name": "get_weather", "input": {}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "tu1", "content": "sunny"}], + }, + ], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert data["tools"] == original_tools, ( + "tools the guardrail synthesized without seeing the request's tools must not replace them" + ) + @pytest.mark.asyncio async def test_guardrail_is_not_called_when_the_request_has_no_tool_results(self): handler = AnthropicMessagesHandler() diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 907da66e5bf..269afef69cd 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1253,6 +1253,31 @@ class StructuredRedactionGuardrail(CustomGuardrail): return inputs +class ToolSynthesizingGuardrail(CustomGuardrail): + """Appends its own function tool to whatever tools it was given, like a + retrieval/recovery guardrail that injects a tool the model can later call.""" + + def __init__(self): + super().__init__(guardrail_name="tool-synthesizing") + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + tools = list(inputs.get("tools") or []) + tools.append( + { + "type": "function", + "function": {"name": "injected_retrieve", "parameters": {"type": "object", "properties": {}}}, + } + ) + inputs["tools"] = tools + return inputs + + class TestScanOnlyToolResults: def _bedrock_guardrail(self): from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail @@ -1348,6 +1373,35 @@ class TestScanOnlyToolResults: "function definitions must stay out of a tool-results-only scan" ) + @pytest.mark.parametrize("scan_only_tool_results", [True, False]) + @pytest.mark.asyncio + async def test_guardrail_synthesized_tools_never_replace_scoped_out_request_tools(self, scan_only_tool_results): + handler = OpenAIChatCompletionsHandler() + guardrail = ToolSynthesizingGuardrail() + guardrail.scan_only_tool_results = scan_only_tool_results + original_tools = [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object", "properties": {}}}, + } + ] + data = { + "messages": [ + {"role": "user", "content": "read the report"}, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}, + ], + "tools": original_tools, + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + if scan_only_tool_results: + assert data["tools"] == original_tools, ( + "tools the guardrail synthesized without seeing the request's tools must not replace them" + ) + else: + assert [t["function"]["name"] for t in data["tools"]] == ["read_file", "injected_retrieve"] + @pytest.mark.asyncio async def test_structured_write_back_keeps_out_of_scope_messages(self): handler = OpenAIChatCompletionsHandler() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 65d6e33588f..76a695ce3fd 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -3670,3 +3670,40 @@ async def test_moderation_hook_honors_the_mcp_event_type(mode, call_type, should "the scan must be logged under the event it actually ran for, so guardrail logs, " "OTel spans, and Langfuse metadata do not misclassify MCP enforcement as an LLM call" ) + + +class TestScanOnlyToolResultsWithLatestRoleFilter: + @pytest.mark.asyncio + async def test_warns_and_skips_when_scoped_payload_has_no_user_message(self): + """scan_only_tool_results hands Bedrock a tool-role-only payload, but + experimental_use_latest_role_message_only scans only the latest user + message: the silent no-op must warn.""" + guardrail = BedrockGuardrail( + guardrail_name="bedrock-latest-role-scoped", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + default_on=True, + experimental_use_latest_role_message_only=True, + ) + guardrail.scan_only_tool_results = True + inputs = { + "texts": ["TOOL-RESULT"], + "structured_messages": [{"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}], + } + + with ( + patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api, + patch( + "litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.verbose_proxy_logger.warning" + ) as mock_warning, + ): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"litellm_call_id": "test-call-id"}, + input_type="request", + ) + + mock_api.assert_not_called() + assert result["texts"] == ["TOOL-RESULT"] + warning_text = " ".join(str(arg) for c in mock_warning.call_args_list for arg in c.args) + assert "scan_only_tool_results" in warning_text diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 431a7aa6f02..2f0fd51539d 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -1696,6 +1696,34 @@ class TestPanwAirsApplyGuardrail: request_data=request_data, guardrail_name=handler.guardrail_name ) + @pytest.mark.asyncio + async def test_apply_guardrail_warns_when_tool_results_scope_leaves_nothing_scannable(self, handler): + """scan_only_tool_results hands PANW a tool-role-only payload, but PANW's role + filter only scans user/system/developer rows: the silent no-op must warn.""" + handler.scan_only_tool_results = True + inputs: GenericGuardrailAPIInputs = { + "texts": ["TOOL-RESULT"], + "structured_messages": [{"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}], + } + request_data = {"litellm_call_id": "test-call-id", "model": "gpt-4"} + + with ( + patch.object(handler, "_call_panw_api", new_callable=AsyncMock) as mock_api, + patch( + "litellm.proxy.guardrails.guardrail_hooks.panw_prisma_airs.panw_prisma_airs.verbose_proxy_logger.warning" + ) as mock_warning, + ): + result = await handler.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + mock_api.assert_not_called() + assert result["texts"] == ["TOOL-RESULT"] + warning_text = " ".join(str(arg) for c in mock_warning.call_args_list for arg in c.args) + assert "scan_only_tool_results" in warning_text + @pytest.mark.asyncio async def test_apply_guardrail_block(self, handler): """Test block action raises HTTPException(400).""" From fa47c47020cb702cd28bf98cde3b924ae4b3d491 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:33:24 -0700 Subject: [PATCH 051/672] fix(lint): measure the basedpyright budget gate in a gate-owned venv The gate previously measured whatever environment the caller happened to have. Locally that is the fat bootstrap venv (--extra proxy pulls in fastapi-sso, whose type info flips a reportUnnecessaryIsInstance diagnostic in ui_sso.py), while CI's publisher venv only has the proxy-dev and e2e-dev groups, so identical trees measured 866 locally vs 865 in CI and every local gate run breached by a phantom +1 scripts/type_check_gate.py now provisions .venv-typecheck itself: a frozen uv sync of the canonical proxy-dev and e2e-dev groups, the interpreter pinned to pyrightconfig.json's pythonVersion, plus the generated Prisma client. Every measurement pass is pinned to that env with --pythonpath, because basedpyright auto-detects a .venv in the project root and that auto-detection beats both PATH order and VIRTUAL_ENV, so the CLI flag is the only pin that actually works. The dependency-group set is folded into the environment fingerprint, so artifacts or caches recorded under a different group set never match and the gate falls back to computing base counts locally instead of comparing mismatched environments The publisher workflow drops its own install and prisma steps and lets the script build the measurement env, and the node heap for the full-tree pass drops from 12GB to 8GB (peak RSS measured at 5.4GB) --- .../publish-basedpyright-base-counts.yml | 18 +-- .gitignore | 1 + Makefile | 8 +- scripts/type_check_gate.py | 118 +++++++++++++++--- tests/test_litellm/test_type_check_gate.py | 99 +++++++++++++-- 5 files changed, 203 insertions(+), 41 deletions(-) diff --git a/.github/workflows/publish-basedpyright-base-counts.yml b/.github/workflows/publish-basedpyright-base-counts.yml index d9b034684a4..c85d30df0ce 100644 --- a/.github/workflows/publish-basedpyright-base-counts.yml +++ b/.github/workflows/publish-basedpyright-base-counts.yml @@ -43,22 +43,14 @@ jobs: with: version: "0.10.9" - - name: Install dependencies - run: | - uv sync --frozen --group proxy-dev --group e2e-dev - - # Mirrors test-linting.yml's lint job: basedpyright resolves Prisma's - # generated client only after `prisma generate`, and the published counts - # must match what that job would measure for the same tree. - - name: Generate Prisma client + # The gate provisions its own measurement env (.venv-typecheck: a frozen + # uv sync of its canonical dependency groups plus a generated Prisma + # client), so no install step here can drift from what local runs measure. + - name: Emit basedpyright counts for HEAD env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | - uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Emit basedpyright counts for HEAD - run: | - uv run --no-sync python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts" + python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts" counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json) echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV" diff --git a/.gitignore b/.gitignore index 13f2202305d..3329f39ca10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .python-version .venv +.venv-typecheck .venv_policy_test .env .claude diff --git a/Makefile b/Makefile index 3e82e141c77..493828571b7 100644 --- a/Makefile +++ b/Makefile @@ -124,10 +124,10 @@ lint-fetch-base: git fetch origin litellm_internal_staging # Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated -# Prisma client, so basedpyright resolves the same modules CI does (without the generated -# client the DB wrappers typed against it degrade to Unknown, drifting the budget from -# CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the -# running proxy need. +# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The +# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its +# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras +# gen:api and the running proxy need. lint-install: $(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev $(UV_RUN) python scripts/prisma_generate_if_needed.py diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index b8c6cb29a4e..f9d7f2912fd 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -12,6 +12,18 @@ a red once two PRs each land near the limit and their sum crosses it: the bystander's count equals its base, so it is spared, while any PR that actually grows the rule past its limit still fails. +Installed packages are part of the measurement: a typed dependency that is +present changes what basedpyright can prove (and therefore which diagnostics +fire) versus when it is absent, so counts from two differently provisioned +venvs are not comparable and their comparison produces phantom breaches no +diff hunk explains. The gate therefore provisions its own environment at +``.venv-typecheck`` (a frozen ``uv sync`` of one canonical dependency-group +set, plus a generated Prisma client) and runs every basedpyright pass from it, +so pre-commit, the CI lint job, and the artifact publisher measure one package +set by construction; re-syncs of an up-to-date env are a near-instant no-op. +The group set is folded into the cache and artifact fingerprint, so counts +recorded under a different set are never matched, only recomputed. + The gate runs basedpyright itself, for both the head and the base pass, with ``NODE_OPTIONS`` raised to the heap this repo needs: basedpyright's node process OOMs at the ~4 GB default, and when callers had to remember the flag, @@ -21,8 +33,8 @@ matters once some rule is over its limit, so when none is the base pass is skipped outright. When it is needed, it is a second basedpyright pass over a detached worktree at the merge-base, run under the same environment so import resolution matches, and its per-rule counts are cached under the repo's git -common dir keyed by merge-base commit, -``pyrightconfig.json``, and ``uv.lock``, so re-runs against the same branch +common dir keyed by merge-base commit, ``pyrightconfig.json``, ``uv.lock``, +and the dependency-group set, so re-runs against the same branch point pay for it once. A CI workflow publishes every staging commit's counts as an artifact (``--emit-counts-dir`` is its entry point), and on a disk-cache miss the gate first tries to download the merge-base's artifact through the ``gh`` @@ -64,10 +76,18 @@ CACHE_FILE_PREFIX = "basedpyright-base-" ARTIFACT_NAME_PREFIX = "basedpyright-counts-" GH_TIMEOUT_SECONDS = 10 +# The one environment every basedpyright pass measures in. The group set is +# the slim one the CI publisher has always installed (not bootstrap's fatter +# --extra proxy env), so the committed budgets stay valid; changing it re-keys +# every cache and artifact fingerprint, so stale counts can never be matched. +TYPECHECK_ENV_DIR = REPO_ROOT / ".venv-typecheck" +TYPECHECK_DEP_GROUPS = ("proxy-dev", "e2e-dev") +PRISMA_GENERATE_SCRIPT = REPO_ROOT / "scripts" / "prisma_generate_if_needed.py" + # basedpyright's node process needs more than the ~4 GB default heap on this # repo; appended last so it wins node's last-flag-wins resolution over any # caller-set value while preserving the caller's other NODE_OPTIONS flags. -NODE_HEAP_OPTION = "--max-old-space-size=12288" +NODE_HEAP_OPTION = "--max-old-space-size=8192" # Bucket for a basedpyright diagnostic with no `rule`. Counted so it's gated. UNCODED = "" @@ -129,14 +149,78 @@ def node_options_with_heap(base_env: Mapping[str, str]) -> str: return f"{base_env.get('NODE_OPTIONS', '')} {NODE_HEAP_OPTION}".strip() -def run_basedpyright(cwd: Path = REPO_ROOT) -> str: - """One basedpyright pass over `cwd` with the raised node heap exported. +def typecheck_python_version() -> str | None: + """The interpreter version to build the owned env with, read from + pyrightconfig's `pythonVersion` so the packages installed for basedpyright + to see always come from the same version it type-checks against.""" + try: + config = json.loads(PYRIGHT_CONFIG.read_text()) + except (OSError, ValueError): + return None + version: Final = config.get("pythonVersion") if isinstance(config, dict) else None + return version if isinstance(version, str) else None - Exit 0 (clean) and 1 (errors found) are both output-bearing runs; anything - else is a crash and fails loudly instead of reading as zero errors.""" - exe = shutil.which("basedpyright") or "basedpyright" + +def typecheck_env_commands(env_dir: Path = TYPECHECK_ENV_DIR) -> tuple[tuple[str, ...], ...]: + python_pin: Final = typecheck_python_version() + sync: Final = ( + "uv", + "sync", + "--frozen", + *(("--python", python_pin) if python_pin else ()), + *(flag for group in TYPECHECK_DEP_GROUPS for flag in ("--group", group)), + ) + generate: Final = (str(env_dir / "bin" / "python"), str(PRISMA_GENERATE_SCRIPT)) + return (sync, generate) + + +def _run_provision_step(cmd: tuple[str, ...], env: Mapping[str, str]) -> int: proc = subprocess.run( - [exe, "--outputjson"], + list(cmd), cwd=REPO_ROOT, env=dict(env), capture_output=True, text=True + ) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + sys.stderr.write(proc.stderr) + return proc.returncode + + +def ensure_typecheck_env( + env_dir: Path = TYPECHECK_ENV_DIR, + run: Callable[[tuple[str, ...], Mapping[str, str]], int] = _run_provision_step, +) -> Path: + """Sync the gate-owned venv (and its generated Prisma client) before a + measurement pass. Unconditional on purpose: an up-to-date env makes both + steps near-instant no-ops, and skipping them on a heuristic is how the + measured environment and the fingerprinted one drift apart.""" + env: Final = {**os.environ, "UV_PROJECT_ENVIRONMENT": str(env_dir)} + for cmd in typecheck_env_commands(env_dir): + if run(cmd, env) != 0: + raise SystemExit( + f"could not provision the type-check environment at {env_dir}: " + f"`{' '.join(cmd)}` failed" + ) + return env_dir + + +def run_basedpyright(cwd: Path = REPO_ROOT, env_dir: Path = TYPECHECK_ENV_DIR) -> str: + """One basedpyright pass over `cwd` from the gate-owned venv, with the + raised node heap exported. + + `--pythonpath` pins import resolution to the owned env's interpreter; it is + the only pin that works, because basedpyright auto-detects a `.venv` in the + project root and that beats both PATH order and VIRTUAL_ENV, silently + measuring the caller's fatter venv (whose extra typed packages flip + diagnostics) whenever the repo has one. Exit 0 (clean) and 1 (errors + found) are both output-bearing runs; anything else is a crash and fails + loudly instead of reading as zero errors.""" + bin_dir: Final = env_dir / "bin" + proc = subprocess.run( + [ + str(bin_dir / "basedpyright"), + "--outputjson", + "--pythonpath", + str(bin_dir / "python"), + ], cwd=cwd, capture_output=True, text=True, @@ -208,11 +292,16 @@ def over_ceiling( ) -def environment_fingerprints() -> tuple[str, ...]: - return tuple( - hashlib.sha256(path.read_bytes()).hexdigest() - for path in (PYRIGHT_CONFIG, UV_LOCK) - if path.exists() +def environment_fingerprints( + dep_groups: tuple[str, ...] = TYPECHECK_DEP_GROUPS, +) -> tuple[str, ...]: + return ( + *( + hashlib.sha256(path.read_bytes()).hexdigest() + for path in (PYRIGHT_CONFIG, UV_LOCK) + if path.exists() + ), + "groups:" + ",".join(dep_groups), ) @@ -560,6 +649,7 @@ def main() -> None: parser.add_argument("--update", action="store_true") parser.add_argument("--emit-counts-dir", type=Path) args = parser.parse_args() + ensure_typecheck_env() head = count_basedpyright(run_basedpyright()) if args.emit_counts_dir is not None: cmd_emit_counts( diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py index e381f787d78..2d9731db53f 100644 --- a/tests/test_litellm/test_type_check_gate.py +++ b/tests/test_litellm/test_type_check_gate.py @@ -1,6 +1,5 @@ import importlib.util import json -import os import subprocess from pathlib import Path @@ -84,33 +83,51 @@ def test_node_options_with_heap_appends_after_caller_flags_so_it_wins(): assert merged == f"--max-old-space-size=4096 --no-warnings {gate.NODE_HEAP_OPTION}" -def _stub_basedpyright(tmp_path, monkeypatch, script_body): - stub = tmp_path / "basedpyright" +def _stub_env(tmp_path, script_body): + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + stub = bin_dir / "basedpyright" stub.write_text(f"#!/bin/sh\n{script_body}\n") stub.chmod(0o755) - monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) + return tmp_path def test_run_basedpyright_exports_the_raised_heap_to_the_child(tmp_path, monkeypatch): captured = tmp_path / "node_options.txt" - _stub_basedpyright( + env_dir = _stub_env( tmp_path, - monkeypatch, f'echo "$NODE_OPTIONS" > "{captured}"\necho \'{{"generalDiagnostics": []}}\'', ) monkeypatch.delenv("NODE_OPTIONS", raising=False) - assert json.loads(gate.run_basedpyright(cwd=tmp_path)) == {"generalDiagnostics": []} + assert json.loads(gate.run_basedpyright(cwd=tmp_path, env_dir=env_dir)) == { + "generalDiagnostics": [] + } assert captured.read_text().strip() == gate.NODE_HEAP_OPTION -def test_run_basedpyright_fails_loudly_on_a_crash_exit_code(tmp_path, monkeypatch): +def test_run_basedpyright_pins_import_resolution_to_the_owned_env(tmp_path): + # basedpyright auto-detects a `.venv` in the project root, and that beats + # PATH order and VIRTUAL_ENV; only an explicit --pythonpath keeps the + # caller's fatter venv (whose extra typed packages flip diagnostics vs CI) + # out of the measurement. + captured = tmp_path / "argv.txt" + env_dir = _stub_env( + tmp_path, + f'echo "$@" > "{captured}"\necho \'{{"generalDiagnostics": []}}\'', + ) + gate.run_basedpyright(cwd=tmp_path, env_dir=env_dir) + argv = captured.read_text().split() + assert argv[argv.index("--pythonpath") + 1] == str(env_dir / "bin" / "python") + + +def test_run_basedpyright_fails_loudly_on_a_crash_exit_code(tmp_path): import pytest # 134 is SIGABRT, what node dies with on a heap OOM; it must never read as a # clean zero-error run. - _stub_basedpyright(tmp_path, monkeypatch, "exit 134") + env_dir = _stub_env(tmp_path, "exit 134") with pytest.raises(SystemExit): - gate.run_basedpyright(cwd=tmp_path) + gate.run_basedpyright(cwd=tmp_path, env_dir=env_dir) def test_at_or_under_ceiling_passes(): @@ -248,6 +265,68 @@ def test_cache_key_changes_with_base_point_and_each_fingerprint(): assert gate.cache_key("abc", ("cfg", "lock2")) != key +def test_fingerprints_carry_the_dependency_group_set(): + # Counts measured under one group set must never be compared against + # another's: the fingerprint difference re-keys every cache entry and + # artifact name, so a changed canonical set falls back to recompute. + assert gate.environment_fingerprints() == gate.environment_fingerprints() + assert gate.environment_fingerprints( + dep_groups=("proxy-dev",) + ) != gate.environment_fingerprints(dep_groups=("proxy-dev", "e2e-dev")) + assert gate.environment_fingerprints()[-1] == "groups:" + ",".join( + gate.TYPECHECK_DEP_GROUPS + ) + + +def test_env_commands_sync_the_canonical_groups_then_generate_prisma(): + sync, generate = gate.typecheck_env_commands(Path("/envdir")) + assert sync[:3] == ("uv", "sync", "--frozen") + adjacent = list(zip(sync, sync[1:])) + for group in gate.TYPECHECK_DEP_GROUPS: + assert ("--group", group) in adjacent + assert generate == ( + str(Path("/envdir") / "bin" / "python"), + str(gate.PRISMA_GENERATE_SCRIPT), + ) + + +def test_env_interpreter_pin_tracks_pyrightconfigs_python_version(): + configured = json.loads((ROOT / "pyrightconfig.json").read_text())[ + "pythonVersion" + ] + assert gate.typecheck_python_version() == configured + sync = gate.typecheck_env_commands()[0] + assert sync[sync.index("--python") + 1] == configured + + +def test_ensure_env_targets_the_owned_dir_and_runs_sync_then_generate(tmp_path): + calls = [] + + def runner(cmd, env): + calls.append((cmd[:2], env["UV_PROJECT_ENVIRONMENT"])) + return 0 + + assert gate.ensure_typecheck_env(env_dir=tmp_path, run=runner) == tmp_path + assert calls == [ + (("uv", "sync"), str(tmp_path)), + ((str(tmp_path / "bin" / "python"), str(gate.PRISMA_GENERATE_SCRIPT)), str(tmp_path)), + ] + + +def test_ensure_env_fails_loudly_and_stops_at_the_first_failed_step(tmp_path): + import pytest + + calls = [] + + def failing(cmd, env): + calls.append(cmd) + return 2 + + with pytest.raises(SystemExit): + gate.ensure_typecheck_env(env_dir=tmp_path, run=failing) + assert len(calls) == 1 + + def test_cached_counts_round_trip(tmp_path): path = gate.cache_path(tmp_path, "abc123", ("f1", "f2")) gate.store_counts(tmp_path, path, "abc123", {"reportAny": 3, "reportCall": 1}) From e2cb01c87cda40b95ac86fe02bec92e7db892112 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 04:37:48 +0000 Subject: [PATCH 052/672] fix(types): correct annotations that were false about their runtime values An adversarial review of the previous commit found annotations that described what the code wished were true rather than what flows through. A false annotation is worse than the Any it replaced, since it launders a wrong assumption past the type checker. - purview: `_resolve_user_id` claimed every request-body value was a Mapping, contradicting `_resolve_trusted_user_id` one method over, which types the same argument `Mapping[str, object]`. `_should_block` claimed every Graph response value was a sequence of str->str mappings and was not assignable from its own producer's return type. - cato: `_CatoAnalyzeResponse.required_action` was required and non-nullable while the API returns null, as seven fixtures in the guardrail's own suite assert. `analysis_result` had the same problem. The streaming hook narrowed an override parameter below what `ProxyLogging` actually passes it. - marketplace: `_PluginRecord.manifest_json` was `str` against a nullable column. Making it honest surfaced a latent crash, covered below. - ownership: two functions took an attribute Protocol while their own bodies branch on `isinstance(response, dict)`, which no Protocol can satisfy. - openapi generator: `paths` claimed every path-item value was an operation, though path items also carry `parameters`, `summary` and `$ref`. - custom openapi spec: a TypedDict asserted a shape that the function returns raw Pydantic sub-schemas out of. Reverted to Any, which is imprecise but not false. `get_marketplace` did an unguarded `json.loads` on the nullable `manifest_json` inside an `except json.JSONDecodeError`, which cannot catch the TypeError a NULL raises, so one NULL row 500s the endpoint. It now skips the plugin like the file's other two read sites already do, with a regression test that fails without the guard. Where honesty cost precision, precision lost. `_should_block` went back to its original signature entirely: the narrowing needed to type it turned a fail-closed DLP control fail-open, because the TypeError it used to raise on a malformed response reached `except Exception` and became a 400. --- litellm/integrations/galileo.py | 17 +++++-- .../mcp_server/openapi_to_mcp_generator.py | 46 +++++++++---------- .../claude_code_marketplace.py | 6 +-- .../proxy/common_utils/custom_openapi_spec.py | 9 +--- .../proxy/container_endpoints/ownership.py | 24 ++++------ .../cato_networks/cato_networks.py | 26 ++++++----- .../guardrail_hooks/microsoft_purview/base.py | 13 +++--- .../test_claude_code_marketplace.py | 27 ++++++++++- 8 files changed, 98 insertions(+), 70 deletions(-) diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index f9ec825e922..2c9ac63941c 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -6,7 +6,7 @@ import re import uuid from collections.abc import Mapping, Sequence from datetime import datetime, timezone, tzinfo -from typing import Any, Final, cast +from typing import Any, Final, TypedDict, cast import httpx from pydantic import BaseModel, Field @@ -35,6 +35,17 @@ GALILEO_CLOUD_API_BASE_URL: Final = "https://api.galileo.ai" GALILEO_MAX_IN_MEMORY_RECORDS: Final = 1000 +class GalileoStandardLoggingFields(TypedDict, total=False): + call_type: str + model: str + prompt_tokens: int + completion_tokens: int + total_tokens: int + response_cost: float + startTime: float + endTime: float + + class LLMResponse(BaseModel): latency_ms: int status_code: int @@ -60,7 +71,7 @@ class LLMResponse(BaseModel): class GalileoObserve(CustomLogger): def __init__(self) -> None: - self.in_memory_records: list[dict[str, Any]] = [] + self.in_memory_records: list[Mapping[str, object]] = [] self.batch_size = 1 self.api_key = os.getenv("GALILEO_API_KEY") self.project_id = os.getenv("GALILEO_PROJECT_ID") @@ -648,7 +659,7 @@ class GalileoObserve(CustomLogger): ) return - slo: Final[Mapping[str, Any] | None] = kwargs.get("standard_logging_object") + slo: Final[GalileoStandardLoggingFields | None] = kwargs.get("standard_logging_object") if slo is None: verbose_logger.debug("Galileo Logger: no standard_logging_object in kwargs, skipping") return diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 41057840684..2cc761f99ed 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -47,11 +47,7 @@ from litellm.proxy._experimental.mcp_server.tool_registry import ( global_mcp_tool_registry, ) -_OpenAPIObject: TypeAlias = Mapping[str, Any] - - -class _OpenAPIParameterSchema(TypedDict, total=False): - type: str +_OpenAPIParameter: TypeAlias = Mapping[str, Any] class _OpenAPIJSONSchema(TypedDict, total=False): @@ -72,16 +68,18 @@ class _OpenAPIOperation(TypedDict, total=False): operationId: str summary: str description: str - parameters: Sequence[_OpenAPIObject] + parameters: Sequence[_OpenAPIParameter] requestBody: _OpenAPIRequestBody class _OpenAPIPathItem(TypedDict, total=False): - parameters: Sequence[_OpenAPIObject] + summary: str + description: str + parameters: Sequence[_OpenAPIParameter] class _OpenAPIComponents(TypedDict, total=False): - parameters: Mapping[str, _OpenAPIObject] + parameters: Mapping[str, _OpenAPIParameter] # Store the base URL and headers globally @@ -161,7 +159,7 @@ async def load_openapi_spec_async(filepath: str) -> dict[str, Any]: return json.load(f) -def get_base_url(spec: _OpenAPIObject, spec_path: str | None = None) -> str: +def get_base_url(spec: Mapping[str, Any], spec_path: str | None = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: @@ -212,7 +210,9 @@ def get_base_url(spec: _OpenAPIObject, spec_path: str | None = None) -> str: return "" -def _resolve_ref(param: _OpenAPIObject, component_params: Mapping[str, _OpenAPIObject]) -> _OpenAPIObject | None: +def _resolve_ref( + param: _OpenAPIParameter, component_params: Mapping[str, _OpenAPIParameter] +) -> _OpenAPIParameter | None: """Resolve a single parameter, following a $ref if present. Returns the resolved param dict, or None if the $ref target is absent from @@ -226,8 +226,8 @@ def _resolve_ref(param: _OpenAPIObject, component_params: Mapping[str, _OpenAPIO def _resolve_param_list( - raw: Sequence[_OpenAPIObject], component_params: Mapping[str, _OpenAPIObject] -) -> list[_OpenAPIObject]: + raw: Sequence[_OpenAPIParameter], component_params: Mapping[str, _OpenAPIParameter] +) -> list[_OpenAPIParameter]: """Resolve $refs in a parameter list, dropping any unresolvable entries.""" result: Final = [] for p in raw: @@ -256,7 +256,7 @@ def resolve_operation_params( merged with the operation-level params; operation-level wins when the same ``name`` + ``in`` combination appears in both. """ - component_params: Final[Mapping[str, _OpenAPIObject]] = components.get("parameters", {}) + component_params: Final[Mapping[str, _OpenAPIParameter]] = components.get("parameters", {}) path_level: Final = _resolve_param_list(path_item.get("parameters", []), component_params) op_level: Final = _resolve_param_list(operation.get("parameters", []), component_params) op_keys: Final = {(p["name"], p.get("in")) for p in op_level} @@ -266,9 +266,8 @@ def resolve_operation_params( return result -def extract_parameters(operation: _OpenAPIObject) -> tuple[Sequence[str], Sequence[str], Sequence[str]]: +def extract_parameters(operation: Mapping[str, Any]) -> tuple[Sequence[str], Sequence[str], Sequence[str]]: """Extract parameter names from OpenAPI operation.""" - param: _OpenAPIObject path_params: Final = [] query_params: Final = [] body_params: Final = [] @@ -278,7 +277,7 @@ def extract_parameters(operation: _OpenAPIObject) -> tuple[Sequence[str], Sequen for param in operation["parameters"]: if "name" not in param: continue - param_name: str = param["name"] + param_name = param["name"] if param.get("in") == "path": path_params.append(param_name) elif param.get("in") == "query": @@ -293,9 +292,8 @@ def extract_parameters(operation: _OpenAPIObject) -> tuple[Sequence[str], Sequen return path_params, query_params, body_params -def build_input_schema(operation: _OpenAPIObject) -> dict[str, Any]: +def build_input_schema(operation: Mapping[str, Any]) -> dict[str, Any]: """Build MCP input schema from OpenAPI operation.""" - param: _OpenAPIObject properties: Final = {} required: Final = [] @@ -304,9 +302,9 @@ def build_input_schema(operation: _OpenAPIObject) -> dict[str, Any]: for param in operation["parameters"]: if "name" not in param: continue - param_name: str = param["name"] - param_schema: _OpenAPIParameterSchema = param.get("schema", {}) - param_type: str = param_schema.get("type", "string") + param_name = param["name"] + param_schema = param.get("schema", {}) + param_type = param_schema.get("type", "string") properties[param_name] = { "type": param_type, @@ -391,7 +389,7 @@ def _merge_openapi_tool_request_headers( def create_tool_function( path: str, method: str, - operation: _OpenAPIObject, + operation: Mapping[str, Any], base_url: str, headers: dict[str, str] | None = None, ): @@ -492,9 +490,9 @@ def create_tool_function( return tool_function -def register_tools_from_openapi(spec: _OpenAPIObject, base_url: str) -> None: +def register_tools_from_openapi(spec: Mapping[str, Any], base_url: str) -> None: """Register MCP tools from OpenAPI specification.""" - paths: Final[Mapping[str, Mapping[str, _OpenAPIOperation]]] = spec.get("paths", {}) + paths: Final[Mapping[str, Mapping[str, Any]]] = spec.get("paths", {}) used_names: Final = set() for path, path_item in paths.items(): diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index d340ca2ced3..46ee9b0911d 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -47,7 +47,7 @@ class _PluginRecord(Protocol): name: str version: str | None description: str | None - manifest_json: str + manifest_json: str | None enabled: bool created_at: datetime | None updated_at: datetime | None @@ -108,7 +108,7 @@ async def get_marketplace(): plugin_list: Final = [] for plugin in plugins: try: - manifest: Mapping[str, object] = json.loads(plugin.manifest_json) + manifest: Mapping[str, object] = json.loads(plugin.manifest_json or "{}") except json.JSONDecodeError: verbose_proxy_logger.warning("Plugin %s has invalid manifest JSON, skipping", plugin.name) continue @@ -431,7 +431,7 @@ async def get_plugin( detail={"error": f"Plugin '{plugin_name}' not found"}, ) - manifest: Final[Mapping[str, object]] = json.loads(plugin.manifest_json) if plugin.manifest_json else {} + manifest: Final[Mapping[str, object]] = json.loads(plugin.manifest_json or "{}") if plugin.manifest_json else {} return { "id": plugin.id, diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index 3bedb6f720c..bc7b80801fe 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -1,14 +1,9 @@ from collections.abc import Mapping, Sequence -from typing import Any, Final, TypedDict +from typing import Any, Final from litellm._logging import verbose_proxy_logger -class _FieldSchema(TypedDict, total=False): - type: str - anyOf: Sequence["_FieldSchema"] - - class CustomOpenAPISpec: """ Handler for customizing OpenAPI specifications with Pydantic models @@ -198,7 +193,7 @@ class CustomOpenAPISpec: return schema @staticmethod - def _extract_field_schema(field_def: _FieldSchema) -> _FieldSchema: + def _extract_field_schema(field_def: dict[str, Any]) -> dict[str, Any]: """ Extract a simple schema from a Pydantic field definition for parameter display. diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py index c40f4d4fef2..a559ab49cfa 100644 --- a/litellm/proxy/container_endpoints/ownership.py +++ b/litellm/proxy/container_endpoints/ownership.py @@ -40,13 +40,6 @@ class _ManagedObjectTable(Protocol): async def update(self, *, where: Mapping[str, str], data: Mapping[str, str]) -> _ManagedObjectRow | None: ... -class _ContainerListResponse(Protocol): - data: Sequence[object] - first_id: str | None - last_id: str | None - has_more: bool - - CONTAINER_OBJECT_PURPOSE: Final = "container" # 60s LRU/TTL cache absorbs every container access check before it reaches @@ -279,7 +272,8 @@ async def _get_container_owner(original_container_id: str, custom_llm_provider: if prisma_client is None: return None - row: Final[_ManagedObjectRow | None] = await ManagedObjectRepository(prisma_client).table.find_first( + table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table + row: Final[_ManagedObjectRow | None] = await table.find_first( where={ "model_object_id": model_object_id, "file_purpose": CONTAINER_OBJECT_PURPOSE, @@ -315,7 +309,8 @@ async def _get_stored_container_id(original_container_id: str, custom_llm_provid if prisma_client is None: return None - row: Final[_ManagedObjectRow | None] = await ManagedObjectRepository(prisma_client).table.find_first( + table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table + row: Final[_ManagedObjectRow | None] = await table.find_first( where={ "model_object_id": model_object_id, "file_purpose": CONTAINER_OBJECT_PURPOSE, @@ -359,9 +354,7 @@ def _get_container_list_data(response: object) -> Sequence[object] | None: return data if isinstance(data, list) else None -def _set_container_list_data( - response: _ContainerListResponse, data: list[object], removed_filtered_items: bool = False -) -> _ContainerListResponse: +def _set_container_list_data(response: Any, data: list[object], removed_filtered_items: bool = False) -> object: if isinstance(response, dict): response["data"] = data if data: @@ -401,7 +394,8 @@ async def _get_allowed_container_ids( if prisma_client is None: return set() - rows: Final[Sequence[_ManagedObjectRow]] = await ManagedObjectRepository(prisma_client).table.find_many( + table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table + rows: Final[Sequence[_ManagedObjectRow]] = await table.find_many( where={ "file_purpose": CONTAINER_OBJECT_PURPOSE, "created_by": {"in": owner_scopes}, @@ -416,10 +410,10 @@ async def _get_allowed_container_ids( async def filter_container_list_response( - response: _ContainerListResponse, + response: object, user_api_key_dict: UserAPIKeyAuth, custom_llm_provider: str, -) -> _ContainerListResponse: +) -> object: if is_proxy_admin(user_api_key_dict): return response diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 79867d492a1..958f84e18de 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -9,7 +9,7 @@ import contextlib import json import os import ssl -from collections.abc import AsyncGenerator, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Mapping, Sequence from ssl import SSLContext from typing import TYPE_CHECKING, Any, Final @@ -70,9 +70,13 @@ class _CatoRedactedChat(TypedDict, total=False): all_redacted_messages: Sequence[_CatoRedactedMessage] +class _CatoAnalysisResult(TypedDict, total=False): + policy_drill_down: Mapping[str, object] + + class _CatoAnalyzeResponse(TypedDict): - required_action: _CatoRequiredAction - analysis_result: NotRequired[Mapping[str, Mapping[str, object]]] + required_action: NotRequired[_CatoRequiredAction | None] + analysis_result: NotRequired[_CatoAnalysisResult] redacted_chat: NotRequired[_CatoRedactedChat] @@ -202,7 +206,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return [] @staticmethod - def _iter_schema_string_refs(data: dict): + def _iter_schema_string_refs(data: Mapping[str, Any]): """Yield ``(container, key)`` for every non-empty schema string the proxy forwards to the model inside tool/function and structured-output schemas: each ``tools[].function`` and legacy ``functions[]`` entry plus the @@ -244,7 +248,7 @@ class CatoNetworksGuardrail(CustomGuardrail): stack.extend(reversed(node)) @classmethod - def _extra_inspection_sources(cls, data: dict) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: + def _extra_inspection_sources(cls, data: Mapping[str, Any]) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: """Text the proxy forwards to the model outside chat ``messages``: Responses-API ``input`` and ``instructions``, legacy completion ``prompt`` and tool/function/``response_format`` schema strings. Returned @@ -305,8 +309,8 @@ class CatoNetworksGuardrail(CustomGuardrail): def _handle_block_action( self, - analysis_result: Mapping[str, Mapping[str, object]], - required_action: _CatoRequiredAction, + analysis_result: _CatoAnalysisResult, + required_action: Any, ) -> None: detection_message: Final = required_action.get("detection_message", None) verbose_proxy_logger.info( @@ -420,8 +424,8 @@ class CatoNetworksGuardrail(CustomGuardrail): def _handle_block_action_on_output( self, - analysis_result: Mapping[str, Mapping[str, object]], - required_action: _CatoRequiredAction, + analysis_result: _CatoAnalysisResult, + required_action: Any, ) -> None: detection_message: Final = required_action.get("detection_message", None) verbose_proxy_logger.info( @@ -570,7 +574,7 @@ class CatoNetworksGuardrail(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response: AsyncGenerator[object, None], + response: AsyncIterable[object], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: from litellm.proxy.proxy_server import StreamingCallbackError @@ -622,7 +626,7 @@ class CatoNetworksGuardrail(CustomGuardrail): async def forward_the_stream_to_cato( self, websocket: ClientConnection, - response_iter: AsyncGenerator[object, None], + response_iter: AsyncIterable[object], ) -> None: async for chunk in response_iter: if isinstance(chunk, BaseModel): diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py index 80beb90cf27..3f666178970 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py @@ -3,7 +3,7 @@ import time import uuid from collections import OrderedDict from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Any, Final from typing_extensions import NotRequired, TypedDict @@ -270,9 +270,7 @@ class PurviewGuardrailBase: # User ID resolution # ------------------------------------------------------------------ - def _resolve_user_id( - self, data: Mapping[str, Mapping[str, object]], user_api_key_dict: "UserAPIKeyAuth" - ) -> str | None: + def _resolve_user_id(self, data: Mapping[str, object], user_api_key_dict: "UserAPIKeyAuth") -> str | None: """Resolve the Entra user object ID from request data or auth context. Returns the strongest available identity walking down four sources, in @@ -295,7 +293,10 @@ class PurviewGuardrailBase: if hasattr(user_api_key_dict, "end_user_id") and user_api_key_dict.end_user_id: return str(user_api_key_dict.end_user_id) - metadata: Final[Mapping[str, object]] = data.get("metadata") or data.get("litellm_metadata") or {} + metadata_value: Final[object] = data.get("metadata") or data.get("litellm_metadata") or {} + if not isinstance(metadata_value, Mapping): + return None + metadata: Final[Mapping[str, object]] = metadata_value uid = metadata.get("user_api_key_user_id") if uid: return str(uid) @@ -359,7 +360,7 @@ class PurviewGuardrailBase: # ------------------------------------------------------------------ @staticmethod - def _should_block(response: Mapping[str, Sequence[Mapping[str, str]]]) -> bool: + def _should_block(response: dict[str, Any]) -> bool: """Return True if any policyAction requires blocking.""" for action in response.get("policyActions", []): odata_type = action.get("@odata.type", "") diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py index f94cd471a01..18e0f2cb559 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -18,13 +18,14 @@ from litellm.types.proxy.claude_code_endpoints import ( UpdatePluginRequest, ) from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( + get_marketplace, register_plugin, update_plugin, ) def _make_mock_prisma(): - """Stateful prisma mock that supports find_unique, create, and update.""" + """Stateful prisma mock that supports find_unique, find_many, create, and update.""" store: dict = {} mock_client = MagicMock() @@ -34,6 +35,12 @@ def _make_mock_prisma(): async def _find_unique(where): return store.get(where.get("name")) + async def _find_many(where=None): + records = list(store.values()) + if where and "enabled" in where: + return [r for r in records if r.enabled == where["enabled"]] + return records + async def _create(data): record = MagicMock() record.id = "test-id" @@ -52,6 +59,7 @@ def _make_mock_prisma(): return record mock_table.find_unique = AsyncMock(side_effect=_find_unique) + mock_table.find_many = AsyncMock(side_effect=_find_many) mock_table.create = AsyncMock(side_effect=_create) mock_table.update = AsyncMock(side_effect=_update) mock_client.db.litellm_claudecodeplugintable = mock_table @@ -211,6 +219,23 @@ async def test_update_plugin_db_error_maps_to_structured_500(): assert "connection lost" in exc_info.value.detail["error"] +@pytest.mark.asyncio +async def test_get_marketplace_skips_plugin_with_null_manifest(): + await register_plugin( + request=RegisterPluginRequest(name="good-plugin", source=_GIT_SUBDIR_SOURCE, version="1.0.0"), + user_api_key_dict=_USER, + ) + + table = litellm.proxy.proxy_server.prisma_client.db.litellm_claudecodeplugintable + await table.create(data={"name": "null-manifest-plugin", "manifest_json": None, "enabled": True}) + + response = await get_marketplace() + + assert response.status_code == 200 + body = json.loads(response.body) + assert [plugin["name"] for plugin in body["plugins"]] == ["good-plugin"] + + @pytest.mark.asyncio async def test_register_plugin_git_subdir_missing_url(): """git-subdir without url field raises HTTP 400.""" From 4ab7a33d2c296923b0a2486fc8e3783197385238 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 04:50:01 +0000 Subject: [PATCH 053/672] chore(lint): ratchet lint budgets down by what this branch fixed Lowers the committed ceilings so the headroom shrinks by exactly what was cleared instead of leaving stale slack for the next change to spend. basedpyright -653 errors across 48 rules, with reportAny 29204 -> 28842 and reportExplicitAny 9227 -> 9105. Strict ruff -80 violations, led by ANN401 -59. LIT rules -85, led by LIT001 -76. --- basedpyright-code-budget.json | 16 ++++++++-------- ruff-strict-budget.json | 18 +++++++++--------- type-discipline-budget.json | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 27d96e415fd..62e2ef2c1df 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29204 + "limit": 28842 }, "reportArgumentType": { "limit": 2635 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 9227 + "limit": 9105 }, "reportFunctionMemberAccess": { "limit": 7 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5850 + "limit": 5843 }, "reportMissingTypeArgument": { - "limit": 15833 + "limit": 15816 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45242 + "limit": 45207 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40340 + "limit": 40297 }, "reportUnknownParameterType": { - "limit": 20293 + "limit": 20272 }, "reportUnknownVariableType": { - "limit": 31796 + "limit": 31750 }, "reportUnnecessaryCast": { "limit": 122 diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 421b424757b..81f4a8b97fb 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,30 +1,30 @@ { "ANN001": { - "limit": 3126 + "limit": 3121 }, "ANN002": { "limit": 71 }, "ANN003": { - "limit": 836 + "limit": 834 }, "ANN201": { - "limit": 2037 + "limit": 2033 }, "ANN202": { - "limit": 869 + "limit": 865 }, "ANN204": { - "limit": 715 + "limit": 713 }, "ANN205": { - "limit": 115 + "limit": 114 }, "ANN206": { "limit": 133 }, "ANN401": { - "limit": 1689 + "limit": 1630 }, "ASYNC230": { "limit": 11 @@ -222,7 +222,7 @@ "limit": 0 }, "RET504": { - "limit": 178 + "limit": 177 }, "RUF010": { "limit": 0 @@ -306,7 +306,7 @@ "limit": 0 }, "TID251": { - "limit": 1242 + "limit": 1240 }, "TRY002": { "limit": 528 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index e26ce54ede7..4db4c61ef4b 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 23343 + "limit": 23267 }, "LIT002": { "limit": 27213 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16802 + "limit": 16793 }, "LIT011": { "limit": 5602 From e34b47f682d2d0183685e5d44058a3cf62d09b6e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:11:21 -0700 Subject: [PATCH 054/672] docs: cap all GitHub comments at 15-25 words, curb semicolon splices The 15-25 word cap previously applied only to replies/rebuttals to AI PR review bots; it now covers every GitHub comment (issue comments and PR discussion comments included). The public-writing punctuation bullet also gains a warning that a word cap is not a one-sentence cap, so tight budgets should be met with period splits or conjunctions rather than ";" splices, at most one ";" per message --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1bc4d108da7..926d3e8698a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We pref If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y: - don't use emojis -- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y +- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap is not a one-sentence cap: when writing under tight word budgets (tldrs, 15-25 word review replies), prefer a period split or a conjunction over ";", and keep to at most one ";" per message - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "." @@ -59,7 +59,7 @@ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages When working on a PR, keep the PR description in sync with new commits being made -Replies/rebuttals to AI PR review bots must be 15-25 word human-readable replies +All GitHub comments (issue comments, PR discussion comments, and replies/rebuttals to AI PR review bots) must be 15-25 word human-readable messages, 25 words max Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in From 4f7d1fce3a1b6f469c9809140c824d83895c535e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:13:15 -0700 Subject: [PATCH 055/672] fix(proxy): fall back to the SDK when a queued response's deployment is missing --- .../common_utils/check_responses_cost.py | 2 +- .../test_check_responses_cost.py | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 5a587de12e9..27837b0b5e4 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -52,7 +52,7 @@ class CheckResponsesCost: live in the config; the row then never leaves ``queued``. """ model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) - if model_id is None: + if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None: return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) router_response = await self.llm_router.aget_responses( response_id=response_id, litellm_metadata=litellm_metadata diff --git a/tests/proxy_unit_tests/test_check_responses_cost.py b/tests/proxy_unit_tests/test_check_responses_cost.py index 16ad5c07919..1faf8692b46 100644 --- a/tests/proxy_unit_tests/test_check_responses_cost.py +++ b/tests/proxy_unit_tests/test_check_responses_cost.py @@ -627,6 +627,65 @@ class TestCheckResponsesCost: mock_sdk_aget.assert_called_once() mock_llm_router.aget_responses.assert_not_called() + @pytest.mark.asyncio + async def test_missing_deployment_falls_back_to_sdk( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """ + An encoded id whose deployment was removed from the router must fall back + to the SDK so provider env credentials can still retrieve it, instead of + failing every poll cycle until stale expiration. + """ + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", + model_id="deployment-deleted", + response_id="resp_upstream_789", + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-missing-deployment" + mock_job.file_object = {"model": "gpt-5", "id": encoded_response_id} + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[mock_job] + ) + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + mock_llm_router.get_deployment = MagicMock(return_value=None) + mock_llm_router.aget_responses = AsyncMock( + side_effect=AssertionError("router has no deployment for this model_id") + ) + + mock_response = ResponsesAPIResponse( + id=encoded_response_id, + object="response", + status="completed", + created_at=int(datetime.now().timestamp()), + output=[], + usage=None, + ) + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + mock_sdk_aget.return_value = mock_response + await check_responses_cost_instance.check_responses_cost() + + mock_llm_router.get_deployment.assert_called_once_with(model_id="deployment-deleted") + mock_llm_router.aget_responses.assert_not_called() + mock_sdk_aget.assert_called_once() + assert mock_sdk_aget.call_args[1]["response_id"] == encoded_response_id + + calls = ( + mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list + ) + assert len(calls) == 1 + assert calls[0][1]["data"]["status"] == "completed" + assert calls[0][1]["where"]["id"]["in"] == ["job-missing-deployment"] + @pytest.mark.asyncio async def test_check_responses_cost_with_incomplete_response( self, check_responses_cost_instance, mock_prisma_client From 3e5dee7317631d0a24b0e799485ab5b2103166a3 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:21:18 -0700 Subject: [PATCH 056/672] chore: make it clearer to Claude that GitHub comments must be concise and to use fewer semicolons --- CLAUDE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 926d3e8698a..4d66297e0a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We pref If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y: - don't use emojis -- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap is not a one-sentence cap: when writing under tight word budgets (tldrs, 15-25 word review replies), prefer a period split or a conjunction over ";", and keep to at most one ";" per message +- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "." @@ -41,7 +41,7 @@ Python max line length is 120, not 88 When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing -`make pre-commit` always saves its complete output to a per-worktree log file and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice, and re-run only after the working tree actually changed +`make pre-commit` saves its complete output to a log file in .git (overwriting previous pre-commit logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in @@ -59,7 +59,7 @@ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages When working on a PR, keep the PR description in sync with new commits being made -All GitHub comments (issue comments, PR discussion comments, and replies/rebuttals to AI PR review bots) must be 15-25 word human-readable messages, 25 words max +All GitHub comments must be human-readable and 15-25 word max Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in From 54e9964eb829b242db45d99d3f6214262b7ae56f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 5 Aug 2026 23:22:11 -0700 Subject: [PATCH 057/672] fix(proxy): stop resolving the UI session sentinel team on /search_tools/list Every Admin UI session key is stamped with the reserved team id `litellm-dashboard`, which never has a row in LiteLLM_TeamTable, so the team lookup in _filter_visible_search_tools raised 404 and the endpoint returned 500 for every non-admin dashboard session. Skip the lookup for that sentinel and scope the caller by its key-level allowlist alone, matching how MCP and agent permission checks already treat it. A real team id is still resolved, and a genuine lookup failure now surfaces with its own status instead of being masked as a 500. --- .../search_tool_management.py | 66 ++++-- .../test_search_tool_management.py | 194 ++++++++++++++++++ 2 files changed, 241 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/search_endpoints/search_tool_management.py b/litellm/proxy/search_endpoints/search_tool_management.py index 381a5e14ca2..69edf681e4d 100644 --- a/litellm/proxy/search_endpoints/search_tool_management.py +++ b/litellm/proxy/search_endpoints/search_tool_management.py @@ -2,13 +2,15 @@ CRUD ENDPOINTS FOR SEARCH TOOLS """ +from collections.abc import Awaitable, Callable from datetime import datetime -from typing import Any, Final +from typing import Any, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from litellm._logging import verbose_proxy_logger +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID from litellm.proxy._types import ( LiteLLM_TeamTable, LitellmUserRoles, @@ -46,9 +48,46 @@ def _convert_datetime_to_str(value: datetime | str | None) -> str | None: return value +TeamObjectLookup: TypeAlias = Callable[[str, UserAPIKeyAuth], Awaitable[LiteLLM_TeamTable]] + + +async def _team_object_from_db(team_id: str, user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_TeamTable: + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + return await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + + +def _allowlist_team_id(user_api_key_dict: UserAPIKeyAuth) -> str | None: + """ + The team whose object_permission allowlist scopes this caller, or None when there is none. + + Every Admin UI session key is stamped with UI_SESSION_TOKEN_TEAM_ID, a reserved sentinel that + never has a row in LiteLLM_TeamTable (`/team/new` rejects it as a real team id), so looking it + up would raise 404 instead of resolving a team. It carries no allowlist of its own, so the + caller is scoped by its key-level allowlist alone. Any other team id is looked up for real and + a failed lookup still surfaces. + """ + team_id: Final = user_api_key_dict.team_id + if not team_id or team_id == UI_SESSION_TOKEN_TEAM_ID: + return None + return team_id + + async def _filter_visible_search_tools( search_tools: list[SearchToolInfoResponse], user_api_key_dict: UserAPIKeyAuth, + lookup_team_object: TeamObjectLookup = _team_object_from_db, ) -> list[SearchToolInfoResponse]: """ Drop search tools the caller is not authorized to invoke, applying the same @@ -60,25 +99,12 @@ async def _filter_visible_search_tools( ): return search_tools - from litellm.proxy.auth.auth_checks import ( - can_user_view_search_tool, - get_team_object, - ) - from litellm.proxy.proxy_server import ( - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) + from litellm.proxy.auth.auth_checks import can_user_view_search_tool - team_object: LiteLLM_TeamTable | None = None - if user_api_key_dict.team_id: - team_object = await get_team_object( - team_id=user_api_key_dict.team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=user_api_key_dict.parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) + allowlist_team_id: Final = _allowlist_team_id(user_api_key_dict) + team_object: Final[LiteLLM_TeamTable | None] = ( + await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None + ) visible: Final[list[SearchToolInfoResponse]] = [] for tool in search_tools: @@ -213,6 +239,8 @@ async def list_search_tools( visible_search_tools: Final = await _filter_visible_search_tools(search_tool_configs, user_api_key_dict) return ListSearchToolsResponse(search_tools=visible_search_tools) + except HTTPException: + raise except Exception as e: verbose_proxy_logger.exception("Error getting search tools: %s", e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py b/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py index f2ccfcd0155..158dee7cb78 100644 --- a/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py +++ b/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py @@ -5,6 +5,7 @@ from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient sys.path.insert( @@ -815,3 +816,196 @@ async def test_list_search_tools_admin_with_restricted_key_still_sees_all(): assert response.status_code == 200 names = {t["search_tool_name"] for t in response.json()["search_tools"]} assert names == {"db-tool-1", "db-tool-2", "db-tool-3"} + + +def _search_tool_responses(*names): + from litellm.types.search import SearchToolInfoResponse + + return [ + SearchToolInfoResponse( + search_tool_id=f"id-{name}", + search_tool_name=name, + litellm_params={"search_provider": "perplexity"}, + search_tool_info=None, + created_at=None, + updated_at=None, + is_from_config=False, + ) + for name in names + ] + + +def _recording_team_lookup(team_object=None, raises=None): + """A `TeamObjectLookup` double that records the team ids it was asked to resolve.""" + asked_for = [] + + async def _lookup(team_id, user_api_key_dict): + asked_for.append(team_id) + if raises is not None: + raise raises + return team_object + + return _lookup, asked_for + + +@pytest.mark.asyncio +async def test_list_search_tools_dashboard_session_key_does_not_look_up_the_ui_team(): + """ + Regression: the Admin UI session key is stamped with the reserved team id + ``litellm-dashboard``, which has no row in LiteLLM_TeamTable. Resolving it as a real team + raised 404, which the endpoint reported as a 500, so the Search Tools page was broken for + every non-admin browsing the dashboard. + """ + from litellm.constants import UI_SESSION_TOKEN_TEAM_ID + + dashboard_session_user = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="internal_user", + team_id=UI_SESSION_TOKEN_TEAM_ID, + ) + ui_team_is_not_a_real_team = AsyncMock( + side_effect=HTTPException( + status_code=404, + detail={"error": f"Team doesn't exist in db. Team={UI_SESSION_TOKEN_TEAM_ID}."}, + ) + ) + + with ( + _mock_search_tool_backend(_scoping_db_tools()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + ui_team_is_not_a_real_team, + ), + _override_auth(dashboard_session_user), + ): + response = TestClient(app).get("/search_tools/list") + + assert response.status_code == 200 + names = {t["search_tool_name"] for t in response.json()["search_tools"]} + assert names == {"db-tool-1", "db-tool-2", "db-tool-3"} + ui_team_is_not_a_real_team.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_filter_visible_search_tools_dashboard_session_still_honors_key_allowlist(): + """ + Skipping the synthetic team must not widen visibility: a dashboard session whose key + carries a search_tools allowlist stays scoped to it. + """ + from litellm.constants import UI_SESSION_TOKEN_TEAM_ID + from litellm.proxy.search_endpoints.search_tool_management import ( + _filter_visible_search_tools, + ) + + restricted_dashboard_session = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="internal_user", + team_id=UI_SESSION_TOKEN_TEAM_ID, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-key", + search_tools=["db-tool-3"], + ), + ) + lookup, asked_for = _recording_team_lookup() + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2", "db-tool-3"), + restricted_dashboard_session, + lookup, + ) + + assert [t["search_tool_name"] for t in visible] == ["db-tool-3"] + assert asked_for == [] + + +@pytest.mark.asyncio +async def test_filter_visible_search_tools_still_applies_a_real_team_allowlist(): + """A caller with a real team is still resolved and scoped by that team's allowlist.""" + from litellm.proxy.search_endpoints.search_tool_management import ( + _filter_visible_search_tools, + ) + + team_member = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="internal_user", + team_id="team-1", + ) + lookup, asked_for = _recording_team_lookup( + team_object=LiteLLM_TeamTable( + team_id="team-1", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-team", + search_tools=["db-tool-2"], + ), + ) + ) + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2", "db-tool-3"), + team_member, + lookup, + ) + + assert [t["search_tool_name"] for t in visible] == ["db-tool-2"] + assert asked_for == ["team-1"] + + +@pytest.mark.asyncio +async def test_filter_visible_search_tools_propagates_a_real_team_lookup_failure(): + """ + A caller whose real team cannot be resolved must not fall through to "no team", which + would drop that team's allowlist and show tools the caller may not call. + """ + from litellm.proxy.search_endpoints.search_tool_management import ( + _filter_visible_search_tools, + ) + + team_member = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="internal_user", + team_id="deleted-team", + ) + lookup, asked_for = _recording_team_lookup( + raises=HTTPException(status_code=404, detail={"error": "Team doesn't exist in db."}) + ) + + with pytest.raises(HTTPException) as exc_info: + await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + team_member, + lookup, + ) + + assert exc_info.value.status_code == 404 + assert asked_for == ["deleted-team"] + + +@pytest.mark.asyncio +async def test_list_search_tools_reports_a_missing_real_team_as_404(): + """ + The endpoint surfaces a genuine team lookup failure with its own status instead of + masking it as a 500 or quietly returning an unscoped list. + """ + team_member = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="internal_user", + team_id="deleted-team", + ) + + with ( + _mock_search_tool_backend(_scoping_db_tools()), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + AsyncMock( + side_effect=HTTPException( + status_code=404, + detail={"error": "Team doesn't exist in db. Team=deleted-team."}, + ) + ), + ), + _override_auth(team_member), + ): + response = TestClient(app).get("/search_tools/list") + + assert response.status_code == 404 + assert "search_tools" not in response.json() From eea292abbabed6a9b4bb86630c60efa74fd7e9a5 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 5 Aug 2026 23:24:37 -0700 Subject: [PATCH 058/672] fix(proxy): allow non-admins to reach /user/daily/activity/aggregated The aggregated route was missing from LiteLLMRoutes.self_managed_routes while its paginated sibling /user/daily/activity was listed, so auth rejected every internal user with a 401 before the handler ran. That route backs the default "Your Usage" view in the dashboard, which left the main Usage page broken for non-admin users. The handler already self-scopes: it checks admin view first, then falls back to require_caller_user_id_for_non_admin, defaults a missing user_id to the caller's own, and returns 403 when a non-admin asks for someone else's data. Listing the route restores reachability without widening what a caller can read. check_route_access matches exactly (plus explicit wildcards), so the parent entry never covered the /aggregated sub-path. --- litellm/proxy/_types.py | 1 + .../proxy/auth/test_route_checks.py | 52 ++++++++++++++ .../test_internal_user_endpoints.py | 69 +++++++++++++++++++ 3 files changed, 122 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7bc8ed59a6a..5b1134650f2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -781,6 +781,7 @@ class LiteLLMRoutes(enum.Enum): "/model/update", "/model/delete", "/user/daily/activity", + "/user/daily/activity/aggregated", "/user/available_roles", # read-only role metadata; any authenticated user may read "/user/list", # org admins checked in endpoint; non-admins get 403 "/model/{model_id}/update", diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 9285b997efc..0bfb10320f7 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -3246,3 +3246,55 @@ def test_internal_user_still_blocked_from_another_users_info(): assert exc_info.value.status_code == 403 assert "key not allowed to access this user's info" in str(exc_info.value.detail) + + +@pytest.mark.parametrize( + "route", + [ + "/user/daily/activity", + "/user/daily/activity/aggregated", + ], +) +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +def test_user_daily_activity_routes_reachable_by_non_admin(route, user_role): + """Both /user/daily/activity and its /aggregated sibling power the default + "Your Usage" dashboard view, and both handlers self-scope to the caller + (_user_has_admin_view -> require_caller_user_id_for_non_admin -> 403 on a + user_id mismatch). self_managed_routes is the ONLY list that grants either + route to a non-admin, so dropping one from it 401s every internal user's + main Usage page before the handler ever runs. + """ + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=user_role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=user_role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + +def test_user_daily_activity_aggregated_not_covered_by_prefix_match(): + """check_route_access is exact-match plus explicit wildcards, so listing the + parent /user/daily/activity does not implicitly cover the /aggregated + sub-path. Pins the reason the sibling needs its own entry. + """ + assert not RouteChecks.check_route_access( + route="/user/daily/activity/aggregated", + allowed_routes=["/user/daily/activity"], + ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index aab9a0b4fd0..056c2d3657a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -2294,6 +2294,75 @@ async def test_get_user_daily_activity_aggregated_admin_global_view(monkeypatch) ) +@pytest.mark.asyncio +async def test_get_user_daily_activity_aggregated_non_admin_cannot_view_other_users( + monkeypatch, +): + """ + Same scoping contract as + test_get_user_daily_activity_non_admin_cannot_view_other_users, on the + aggregated route. Non-admins reach this handler now that the route is in + self_managed_routes, so the 403-on-mismatch and default-to-self behaviour + has to hold here too: opening the route must not widen access. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + get_user_daily_activity_aggregated, + ) + + mock_prisma_client = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + non_admin_key_dict = UserAPIKeyAuth( + user_id="regular-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + # Case 1: Non-admin targets another user's data — 403, helper never reached + with patch( + "litellm.proxy.management_endpoints.internal_user_endpoints.get_daily_activity_aggregated", + new_callable=AsyncMock, + ) as mock_get_daily_agg: + with pytest.raises(HTTPException) as exc_info: + await get_user_daily_activity_aggregated( + start_date="2025-01-01", + end_date="2025-01-31", + model=None, + api_key=None, + user_id="other-user-456", + timezone=None, + user_api_key_dict=non_admin_key_dict, + ) + + assert exc_info.value.status_code == 403 + assert "Non-admin users can only view their own spend data" in str(exc_info.value.detail) + mock_get_daily_agg.assert_not_called() + + # Case 2: Non-admin omits user_id — scoped to their own user_id, not global + mock_response = MagicMock() + with patch( + "litellm.proxy.management_endpoints.internal_user_endpoints.get_daily_activity_aggregated", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_daily_agg: + result = await get_user_daily_activity_aggregated( + start_date="2025-01-01", + end_date="2025-01-31", + model=None, + api_key=None, + user_id=None, + timezone=None, + user_api_key_dict=non_admin_key_dict, + ) + + assert result is mock_response + mock_get_daily_agg.assert_called_once() + assert mock_get_daily_agg.call_args.kwargs["entity_id"] == "regular-user-123" + + @pytest.mark.asyncio async def test_delete_user_cleans_up_created_by_invitation_links(mocker): """ From d5a471b2a7be5b5eb923d156ff8327309313895d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 5 Aug 2026 23:40:16 -0700 Subject: [PATCH 059/672] test(proxy): type the search tool test helpers and record lookups with AsyncMock Replaces the hand-rolled recording double with AsyncMock so the awaited team ids come from await_args_list instead of a mutated list, and annotates the response factory now that SearchToolInfoResponse is imported at module level. --- .../test_search_tool_management.py | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py b/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py index 158dee7cb78..a64397d9818 100644 --- a/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py +++ b/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py @@ -24,6 +24,7 @@ import litellm.proxy.proxy_server as ps # Now we can safely import app from litellm.proxy.proxy_server import app +from litellm.types.search import SearchToolInfoResponse client = TestClient(app) @@ -818,9 +819,7 @@ async def test_list_search_tools_admin_with_restricted_key_still_sees_all(): assert names == {"db-tool-1", "db-tool-2", "db-tool-3"} -def _search_tool_responses(*names): - from litellm.types.search import SearchToolInfoResponse - +def _search_tool_responses(*names: str) -> list[SearchToolInfoResponse]: return [ SearchToolInfoResponse( search_tool_id=f"id-{name}", @@ -835,17 +834,8 @@ def _search_tool_responses(*names): ] -def _recording_team_lookup(team_object=None, raises=None): - """A `TeamObjectLookup` double that records the team ids it was asked to resolve.""" - asked_for = [] - - async def _lookup(team_id, user_api_key_dict): - asked_for.append(team_id) - if raises is not None: - raise raises - return team_object - - return _lookup, asked_for +def _team_ids_looked_up(lookup: AsyncMock) -> list[str]: + return [awaited.args[0] for awaited in lookup.await_args_list] @pytest.mark.asyncio @@ -906,7 +896,7 @@ async def test_filter_visible_search_tools_dashboard_session_still_honors_key_al search_tools=["db-tool-3"], ), ) - lookup, asked_for = _recording_team_lookup() + lookup = AsyncMock() visible = await _filter_visible_search_tools( _search_tool_responses("db-tool-1", "db-tool-2", "db-tool-3"), @@ -915,7 +905,7 @@ async def test_filter_visible_search_tools_dashboard_session_still_honors_key_al ) assert [t["search_tool_name"] for t in visible] == ["db-tool-3"] - assert asked_for == [] + lookup.assert_not_awaited() @pytest.mark.asyncio @@ -930,8 +920,8 @@ async def test_filter_visible_search_tools_still_applies_a_real_team_allowlist() user_id="internal_user", team_id="team-1", ) - lookup, asked_for = _recording_team_lookup( - team_object=LiteLLM_TeamTable( + lookup = AsyncMock( + return_value=LiteLLM_TeamTable( team_id="team-1", object_permission=LiteLLM_ObjectPermissionTable( object_permission_id="op-team", @@ -947,7 +937,7 @@ async def test_filter_visible_search_tools_still_applies_a_real_team_allowlist() ) assert [t["search_tool_name"] for t in visible] == ["db-tool-2"] - assert asked_for == ["team-1"] + assert _team_ids_looked_up(lookup) == ["team-1"] @pytest.mark.asyncio @@ -965,9 +955,7 @@ async def test_filter_visible_search_tools_propagates_a_real_team_lookup_failure user_id="internal_user", team_id="deleted-team", ) - lookup, asked_for = _recording_team_lookup( - raises=HTTPException(status_code=404, detail={"error": "Team doesn't exist in db."}) - ) + lookup = AsyncMock(side_effect=HTTPException(status_code=404, detail={"error": "Team doesn't exist in db."})) with pytest.raises(HTTPException) as exc_info: await _filter_visible_search_tools( @@ -977,7 +965,7 @@ async def test_filter_visible_search_tools_propagates_a_real_team_lookup_failure ) assert exc_info.value.status_code == 404 - assert asked_for == ["deleted-team"] + assert _team_ids_looked_up(lookup) == ["deleted-team"] @pytest.mark.asyncio From 7d745521bf94331cc5e0ec069e9c2f7d8d443303 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:46:40 -0700 Subject: [PATCH 060/672] fix(guardrails): merge synthesized tools under scan_only_tool_results and reject role-filtered no-op combos at init --- litellm/integrations/custom_guardrail.py | 10 +++ .../chat/guardrail_translation/handler.py | 14 +++- .../base_llm/guardrail_translation/utils.py | 43 +++++++++++- .../chat/guardrail_translation/handler.py | 14 +++- .../guardrail_hooks/bedrock_guardrails.py | 3 + .../panw_prisma_airs/panw_prisma_airs.py | 3 + .../proxy/guardrails/guardrail_registry.py | 12 ++++ .../test_anthropic_guardrail_handler.py | 8 ++- .../test_openai_guardrail_handler.py | 68 +++++++++++++++++-- .../guardrails/test_guardrail_registry.py | 57 ++++++++++++++++ 10 files changed, 216 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 20f3aa430e9..7c8c9aeb248 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -714,6 +714,16 @@ class CustomGuardrail(CustomLogger): return result + def supports_scan_only_tool_results(self) -> bool: + """Whether this guardrail can scan tool-result content. + + Guardrails whose own role filtering only ever scans human-authored + messages override this to return False, so configuring them with + ``scan_only_tool_results`` is rejected at initialization instead of + silently scanning nothing on every request. + """ + return True + def should_run_guardrail( self, data, diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 184e0f6a343..aef01765e6e 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,10 +26,12 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im ) from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( + anthropic_tool_name, effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, merge_guardrailed_scoped_messages, + merge_returned_tools_into_request_tools, scoped_structured_message_indices, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( @@ -387,7 +389,7 @@ class AnthropicMessagesHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) guardrailed_tools: Final = guardrailed_inputs.get("tools") - if guardrailed_tools is not None and not scan_only_tool_results: + if guardrailed_tools is not None: # Convert tools back from OpenAI format to Anthropic format anthropic_config: Final = AnthropicConfig() anthropic_tools: Final[list[AllAnthropicToolsValues]] = [] @@ -396,7 +398,15 @@ class AnthropicMessagesHandler(BaseTranslation): if converted_tool is not None: anthropic_tools.append(converted_tool) # Note: MCP servers are handled separately in the main transformation - data["tools"] = anthropic_tools + data["tools"] = ( + merge_returned_tools_into_request_tools( + request_tools=data.get("tools"), + returned_tools=anthropic_tools, + tool_name=anthropic_tool_name, + ) + if scan_only_tool_results + else anthropic_tools + ) guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages") if ( diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 432ac64b456..bdfe15ca9a6 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -1,8 +1,8 @@ from __future__ import annotations import json -from collections.abc import Iterator, Sequence -from typing import Any, Final +from collections.abc import Callable, Iterator, Sequence +from typing import Any, Final, TypeVar from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues @@ -167,6 +167,45 @@ def scoped_structured_message_indices( ) +ToolT = TypeVar("ToolT") + + +def openai_tool_name(tool: object) -> str | None: + if not isinstance(tool, dict): + return None + function: Final = tool.get("function") + if isinstance(function, dict): + function_name: Final = function.get("name") + return function_name if isinstance(function_name, str) else None + flat_name: Final = tool.get("name") + return flat_name if isinstance(flat_name, str) else None + + +def anthropic_tool_name(tool: object) -> str | None: + name: Final = tool.get("name") if isinstance(tool, dict) else None + return name if isinstance(name, str) else None + + +def merge_returned_tools_into_request_tools( + request_tools: Sequence[ToolT] | None, + returned_tools: Sequence[ToolT], + tool_name: Callable[[ToolT], str | None], +) -> list[ToolT]: + """Union of the request's tools and guardrail-returned tools, keyed by name. + + Under ``scan_only_tool_results`` the guardrail never saw the request's + tools, so a returned list can neither replace them (it would drop every + user-defined function) nor be discarded (it may carry a tool the guardrail + synthesized and told the model to call, like Compresr's retrieve tool). + Keep every request tool and append only returned tools whose names aren't + already taken. + """ + originals: Final = tuple(request_tools or ()) + taken_names: Final = frozenset(name for tool in originals if (name := tool_name(tool)) is not None) + additions: Final = tuple(tool for tool in returned_tools if tool_name(tool) not in taken_names) + return [*originals, *additions] + + def merge_guardrailed_scoped_messages( full_messages: Sequence[AllMessageValues], scoped_indices: Sequence[int], diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index dc2a06d67fc..4f0f69866ad 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -27,6 +27,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, merge_guardrailed_scoped_messages, + merge_returned_tools_into_request_tools, + openai_tool_name, role_out_of_guardrail_scope, scoped_structured_message_indices, ) @@ -143,8 +145,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", []) guardrailed_tools: Final = guardrailed_inputs.get("tools") - if guardrailed_tools is not None and not scan_only_tool_results: - data["tools"] = guardrailed_tools + if guardrailed_tools is not None: + data["tools"] = ( + merge_returned_tools_into_request_tools( + request_tools=tools, + returned_tools=guardrailed_tools, + tool_name=openai_tool_name, + ) + if scan_only_tool_results + else guardrailed_tools + ) guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages") if ( diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 8193069fd82..e9e729fb118 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -405,6 +405,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): grounding.append(block) return grounding + def supports_scan_only_tool_results(self) -> bool: + return self.experimental_use_latest_role_message_only is not True + def _prepare_guardrail_messages_for_role( self, messages: list[AllMessageValues] | None, diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index a96a0070eef..13ced0ac06c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -1564,6 +1564,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): return scannable + def supports_scan_only_tool_results(self) -> bool: + return False + @staticmethod def _get_scannable_text_indices( texts: list[str], diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index e9e61283c1a..15e884c939c 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -14,6 +14,9 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.llms.base_llm.guardrail_translation.utils import ( + effective_scan_only_tool_results_for_guardrail, +) from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrail, ) @@ -493,6 +496,15 @@ class InMemoryGuardrailHandler: "scan_only_tool_results", ): setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) + if ( + effective_scan_only_tool_results_for_guardrail(custom_guardrail_callback) + and not custom_guardrail_callback.supports_scan_only_tool_results() + ): + raise ValueError( + f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results is enabled, but this " + "guardrail's role filtering never scans tool results, so no request content would ever " + "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." + ) configured_run_in_parallel: Final = getattr(litellm_params, "run_in_parallel", None) if configured_run_in_parallel is not None: custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index c7dedff0663..c7a30f7f954 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -883,7 +883,7 @@ class TestAnthropicMessagesScanOnlyToolResults: assert data["messages"][2]["content"][0]["text"] == "sibling POISON text" @pytest.mark.asyncio - async def test_guardrail_synthesized_tools_never_replace_scoped_out_request_tools(self): + async def test_guardrail_synthesized_tools_are_appended_without_replacing_request_tools(self): handler = AnthropicMessagesHandler() guardrail = ToolAppendingGuardrail(guardrail_name="tool-appending") guardrail.scan_only_tool_results = True @@ -912,9 +912,11 @@ class TestAnthropicMessagesScanOnlyToolResults: await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) - assert data["tools"] == original_tools, ( - "tools the guardrail synthesized without seeing the request's tools must not replace them" + assert [t["name"] for t in data["tools"]] == ["get_weather", "injected_tool"], ( + "a tool the guardrail synthesized must reach the model, converted to Anthropic format, " + "without the request's own tools being replaced or dropped" ) + assert data["tools"][0] == original_tools[0] @pytest.mark.asyncio async def test_guardrail_is_not_called_when_the_request_has_no_tool_results(self): diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 269afef69cd..deabee12497 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1278,6 +1278,35 @@ class ToolSynthesizingGuardrail(CustomGuardrail): return inputs +class ToolNameCollidingGuardrail(CustomGuardrail): + """Returns a tool reusing a request tool's name plus a genuinely new tool.""" + + def __init__(self): + super().__init__(guardrail_name="tool-name-colliding") + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + inputs["tools"] = [ + { + "type": "function", + "function": { + "name": "read_file", + "parameters": {"type": "object", "properties": {"hijacked": {"type": "string"}}}, + }, + }, + { + "type": "function", + "function": {"name": "injected_retrieve", "parameters": {"type": "object", "properties": {}}}, + }, + ] + return inputs + + class TestScanOnlyToolResults: def _bedrock_guardrail(self): from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail @@ -1375,7 +1404,9 @@ class TestScanOnlyToolResults: @pytest.mark.parametrize("scan_only_tool_results", [True, False]) @pytest.mark.asyncio - async def test_guardrail_synthesized_tools_never_replace_scoped_out_request_tools(self, scan_only_tool_results): + async def test_guardrail_synthesized_tools_are_appended_without_replacing_request_tools( + self, scan_only_tool_results + ): handler = OpenAIChatCompletionsHandler() guardrail = ToolSynthesizingGuardrail() guardrail.scan_only_tool_results = scan_only_tool_results @@ -1395,12 +1426,35 @@ class TestScanOnlyToolResults: await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) - if scan_only_tool_results: - assert data["tools"] == original_tools, ( - "tools the guardrail synthesized without seeing the request's tools must not replace them" - ) - else: - assert [t["function"]["name"] for t in data["tools"]] == ["read_file", "injected_retrieve"] + assert [t["function"]["name"] for t in data["tools"]] == ["read_file", "injected_retrieve"], ( + "a tool the guardrail synthesized (like a recovery/retrieve tool) must reach the model " + "without the request's own tools being replaced or dropped" + ) + assert data["tools"][0] == original_tools[0] + + @pytest.mark.asyncio + async def test_returned_tool_name_collisions_keep_the_request_schema(self): + handler = OpenAIChatCompletionsHandler() + guardrail = ToolNameCollidingGuardrail() + guardrail.scan_only_tool_results = True + original_read_file = { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object", "properties": {}}}, + } + data = { + "messages": [ + {"role": "user", "content": "read the report"}, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}, + ], + "tools": [original_read_file], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [t["function"]["name"] for t in data["tools"]] == ["read_file", "injected_retrieve"] + assert data["tools"][0] == original_read_file, ( + "a returned tool reusing a request tool's name must not replace the request's schema" + ) @pytest.mark.asyncio async def test_structured_write_back_keeps_out_of_scope_messages(self): diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 6bd109f0f95..3053664ef27 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -558,3 +558,60 @@ def test_reinitialized_judge_guardrail_uses_lazy_router_provider(): finally: for cb_list, snapshot in zip(lists, snapshots): cb_list[:] = snapshot + + +class TestScanOnlyToolResultsInitRefusal: + """A guardrail whose role filtering never scans tool results must be rejected at + initialization when configured with scan_only_tool_results, instead of booting a + proxy that silently scans nothing on every request.""" + + def _initialize(self, name: str, params: dict): + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + return InMemoryGuardrailHandler().initialize_guardrail( + guardrail={"guardrail_name": name, "litellm_params": params}, + ) + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + def test_panw_prisma_airs_with_scan_only_tool_results_is_rejected(self): + with pytest.raises(ValueError, match="never scans tool results"): + self._initialize( + "panw-scan-only-combo", + { + "guardrail": "panw_prisma_airs", + "mode": "pre_call", + "api_key": "test-key", + "profile_name": "test-profile", + "scan_only_tool_results": True, + }, + ) + + def test_bedrock_latest_role_with_scan_only_tool_results_is_rejected(self): + with pytest.raises(ValueError, match="never scans tool results"): + self._initialize( + "bedrock-latest-role-scan-only-combo", + { + "guardrail": "bedrock", + "mode": "pre_call", + "guardrailIdentifier": "gr-1", + "guardrailVersion": "1", + "experimental_use_latest_role_message_only": True, + "scan_only_tool_results": True, + }, + ) + + def test_bedrock_without_latest_role_accepts_scan_only_tool_results(self): + result = self._initialize( + "bedrock-scan-only-ok", + { + "guardrail": "bedrock", + "mode": "pre_call", + "guardrailIdentifier": "gr-1", + "guardrailVersion": "1", + "scan_only_tool_results": True, + }, + ) + assert result is not None From 28ff7f3f0b68d77428f175e54099b9b590226081 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 00:52:16 -0700 Subject: [PATCH 061/672] fix(guardrails): scan function-role results and dedupe returned tools Under scan_only_tool_results, legacy OpenAI function-role messages now count as tool results, and duplicate names among guardrail-returned tools keep only the first occurrence. CustomGuardrail.structured_messages_cover_full_request lets CrowdStrike AIDR declare that its writeback already rebuilds the whole conversation, so handlers install it as-is instead of merging it into the full message list a second time and duplicating out-of-scope rows. Lint budget ceilings ratchet down to match the tree --- basedpyright-code-budget.json | 18 ++--- litellm/integrations/custom_guardrail.py | 13 ++++ .../chat/guardrail_translation/handler.py | 4 +- .../base_llm/guardrail_translation/utils.py | 11 ++- .../chat/guardrail_translation/handler.py | 12 ++- .../crowdstrike_aidr/crowdstrike_aidr.py | 4 + ruff-strict-budget.json | 2 +- .../test_openai_guardrail_handler.py | 78 +++++++++++++++++++ type-discipline-budget.json | 6 +- 9 files changed, 127 insertions(+), 21 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 8a5c78c1f6c..d98e6c6c911 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -9,10 +9,10 @@ "limit": 329 }, "reportAttributeAccessIssue": { - "limit": 516 + "limit": 514 }, "reportCallIssue": { - "limit": 123 + "limit": 117 }, "reportConstantRedefinition": { "limit": 40 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 9226 + "limit": 9225 }, "reportFunctionMemberAccess": { "limit": 7 @@ -99,25 +99,25 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45242 + "limit": 45145 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40339 + "limit": 39881 }, "reportUnknownParameterType": { - "limit": 20293 + "limit": 20258 }, "reportUnknownVariableType": { - "limit": 31796 + "limit": 31429 }, "reportUnnecessaryCast": { "limit": 122 }, "reportUnnecessaryComparison": { - "limit": 702 + "limit": 701 }, "reportUnnecessaryContains": { "limit": 5 @@ -126,7 +126,7 @@ "limit": 864 }, "reportUntypedBaseClass": { - "limit": 72 + "limit": 0 }, "reportUntypedFunctionDecorator": { "limit": 33 diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 7c8c9aeb248..2e91e082bd4 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -724,6 +724,19 @@ class CustomGuardrail(CustomLogger): """ return True + def structured_messages_cover_full_request(self) -> bool: + """Whether returned ``structured_messages`` span the whole request. + + Translation handlers hand guardrails only the in-scope subset of the + conversation and merge a returned ``structured_messages`` list back + into the full request. A guardrail that already rebuilds the complete + conversation itself (like CrowdStrike AIDR with its skip filters + active) overrides this to return True so the handler installs the + returned list as-is instead of merging it a second time, which would + duplicate the out-of-scope messages. + """ + return False + def should_run_guardrail( self, data, diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index aef01765e6e..88db9fae912 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -415,7 +415,9 @@ class AnthropicMessagesHandler(BaseTranslation): ): self._write_back_structured_messages( data, - merge_guardrailed_scoped_messages( + guardrailed_structured_messages + if guardrail_to_apply.structured_messages_cover_full_request() + else merge_guardrailed_scoped_messages( full_messages=full_structured_messages, scoped_indices=scoped_message_indices, guardrailed_scoped=guardrailed_structured_messages, diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index bdfe15ca9a6..f1ddf21cd3c 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -145,7 +145,7 @@ def role_out_of_guardrail_scope( return True if skip_tool_message and role == "tool": return True - return scan_only_tool_results and role != "tool" + return scan_only_tool_results and role not in ("tool", "function") def scoped_structured_message_indices( @@ -198,11 +198,16 @@ def merge_returned_tools_into_request_tools( user-defined function) nor be discarded (it may carry a tool the guardrail synthesized and told the model to call, like Compresr's retrieve tool). Keep every request tool and append only returned tools whose names aren't - already taken. + already taken by a request tool or an earlier returned tool. """ originals: Final = tuple(request_tools or ()) taken_names: Final = frozenset(name for tool in originals if (name := tool_name(tool)) is not None) - additions: Final = tuple(tool for tool in returned_tools if tool_name(tool) not in taken_names) + additions: Final = tuple( + tool + for index, tool in enumerate(returned_tools) + if (name := tool_name(tool)) not in taken_names + and (name is None or all(tool_name(earlier) != name for earlier in returned_tools[:index])) + ) return [*originals, *additions] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 4f0f69866ad..e411dc497fc 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -161,10 +161,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrailed_structured_messages is not None and guardrailed_structured_messages is not original_structured_messages ): - data["messages"] = merge_guardrailed_scoped_messages( - full_messages=structured_messages or [], - scoped_indices=scoped_message_indices, - guardrailed_scoped=guardrailed_structured_messages, + data["messages"] = ( + guardrailed_structured_messages + if guardrail_to_apply.structured_messages_cover_full_request() + else merge_guardrailed_scoped_messages( + full_messages=structured_messages or [], + scoped_indices=scoped_message_indices, + guardrailed_scoped=guardrailed_structured_messages, + ) ) else: # Step 3: Map guardrail responses back to original message structure diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index 15ddd5e3458..b1bf9159607 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -362,6 +362,10 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): tail: Final = guard_output.messages[-num_assistant_messages:] if num_assistant_messages > 0 else [] return [_extract_text_from_message(msg) for msg in tail] + @override + def structured_messages_cover_full_request(self) -> bool: + return effective_skip_system_message_for_guardrail(self) or effective_skip_tool_message_for_guardrail(self) + def _writeback_messages( self, structured_messages: list[AllMessageValues], diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index ea20ac97e07..65c98f6aab3 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -42,7 +42,7 @@ "limit": 81 }, "B010": { - "limit": 192 + "limit": 190 }, "B018": { "limit": 2 diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index deabee12497..2e75f29b1c5 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1307,6 +1307,38 @@ class ToolNameCollidingGuardrail(CustomGuardrail): return inputs +class DuplicateToolReturningGuardrail(CustomGuardrail): + """Returns the same synthesized tool name twice, second copy with a different schema.""" + + def __init__(self): + super().__init__(guardrail_name="duplicate-tool-returning") + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + inputs["tools"] = [ + { + "type": "function", + "function": { + "name": "injected_retrieve", + "parameters": {"type": "object", "properties": {"first": {"type": "string"}}}, + }, + }, + { + "type": "function", + "function": { + "name": "injected_retrieve", + "parameters": {"type": "object", "properties": {"second": {"type": "string"}}}, + }, + }, + ] + return inputs + + class TestScanOnlyToolResults: def _bedrock_guardrail(self): from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail @@ -1351,6 +1383,28 @@ class TestScanOnlyToolResults: scanned = [m["content"] for m in mock_api.call_args.kwargs["messages"]] assert scanned == ["TOOL-RESULT-scanned"] + @pytest.mark.asyncio + async def test_legacy_function_role_results_are_scanned(self): + from unittest.mock import AsyncMock, patch + + handler = OpenAIChatCompletionsHandler() + guardrail = self._bedrock_guardrail() + data = { + "messages": [ + {"role": "user", "content": "USER-PROMPT-not-scanned"}, + {"role": "function", "name": "read_file", "content": "FUNCTION-RESULT-scanned"}, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT-scanned"}, + ] + } + with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api: + mock_api.return_value = {"action": "NONE", "output": [], "outputs": []} + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + assert mock_api.call_count == 1 + scanned = [m["content"] for m in mock_api.call_args.kwargs["messages"]] + assert scanned == ["FUNCTION-RESULT-scanned", "TOOL-RESULT-scanned"], ( + "a tool result sent with the legacy function role must not bypass the scoped scan" + ) + @pytest.mark.parametrize("flag_value", [None, "false", 0, object()]) @pytest.mark.asyncio async def test_scope_narrows_only_when_the_flag_is_actually_true(self, flag_value): @@ -1456,6 +1510,30 @@ class TestScanOnlyToolResults: "a returned tool reusing a request tool's name must not replace the request's schema" ) + @pytest.mark.asyncio + async def test_duplicate_returned_tool_names_keep_only_the_first(self): + handler = OpenAIChatCompletionsHandler() + guardrail = DuplicateToolReturningGuardrail() + guardrail.scan_only_tool_results = True + original_read_file = { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object", "properties": {}}}, + } + data = { + "messages": [ + {"role": "user", "content": "read the report"}, + {"role": "tool", "tool_call_id": "call_1", "content": "TOOL-RESULT"}, + ], + "tools": [original_read_file], + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [t["function"]["name"] for t in data["tools"]] == ["read_file", "injected_retrieve"], ( + "two returned tools sharing a name must not both be forwarded to the provider" + ) + assert data["tools"][1]["function"]["parameters"]["properties"] == {"first": {"type": "string"}} + @pytest.mark.asyncio async def test_structured_write_back_keeps_out_of_scope_messages(self): handler = OpenAIChatCompletionsHandler() diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 37964c27657..8064e63f1aa 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 23337 + "limit": 23332 }, "LIT002": { "limit": 27213 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1092 + "limit": 1091 }, "LIT007": { "limit": 0 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16796 + "limit": 16792 }, "LIT011": { "limit": 5602 From e8245107654a28dc4021de4712503230633e37df Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 01:54:26 -0700 Subject: [PATCH 062/672] fix(lint): generate the prisma client into the gate-owned venv prisma resolves its prisma-client-py generator through a plain /bin/sh PATH lookup, never through the interpreter that ran prisma generate, so the gate's generate step landed the client in whatever venv the caller had on PATH: the owned env never received one, every gate run regenerated, the caller's venv was mutated instead, and any invocation without a venv on PATH (the rewritten publisher workflow) failed outright The generate now runs with the target interpreter's bin directory pinned to the front of the child PATH. The prisma schema joins the environment fingerprint so clientless counts recorded before this commit can never be compared against clientful ones, a cold provision announces itself on stderr instead of sitting silent for two minutes, and the CI gate step reuses the job's prisma binary cache --- .github/workflows/test-linting.yml | 1 + scripts/prisma_generate_if_needed.py | 36 +++++++++++++++---- scripts/type_check_gate.py | 12 +++++-- .../test_prisma_generate_if_needed.py | 29 +++++++++++++++ tests/test_litellm/test_type_check_gate.py | 22 ++++++++++++ 5 files changed, 91 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 5125dd0a354..5e333f2a3ca 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -115,6 +115,7 @@ jobs: - name: Check basedpyright budget (delta vs base) env: GH_TOKEN: ${{ github.token }} + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA" diff --git a/scripts/prisma_generate_if_needed.py b/scripts/prisma_generate_if_needed.py index d2c40adf820..f2d2232b23f 100644 --- a/scripts/prisma_generate_if_needed.py +++ b/scripts/prisma_generate_if_needed.py @@ -9,13 +9,21 @@ client (a fresh or reinstalled prisma package) forces a regenerate even when the stamp matches. The prisma package itself is never imported here: once generated it re-exports the whole client on import, which costs more than the generate this script exists to skip. + +prisma resolves its generator command (``prisma-client-py``) through a plain +PATH lookup, never through the interpreter that invoked ``prisma generate``, +so the generate runs with this interpreter's own bin directory pinned to the +front of PATH; without that pin the client lands in whichever venv the caller +happened to have on PATH (or the generate fails outright when none is). """ import hashlib import importlib.metadata import importlib.util +import os import subprocess import sys +from collections.abc import Callable, Mapping from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent @@ -46,6 +54,25 @@ def client_is_generated() -> bool: ) +def env_with_own_bin_first(base_env: Mapping[str, str]) -> dict[str, str]: + bin_dir = str(Path(sys.executable).parent) + inherited = base_env.get("PATH") + path = os.pathsep.join((bin_dir, inherited)) if inherited else bin_dir + return {**base_env, "PATH": path} + + +def _run_command(cmd: list[str], cwd: Path, env: dict[str, str]) -> int: + return subprocess.run(cmd, cwd=cwd, env=env).returncode + + +def run_generate(run: Callable[[list[str], Path, dict[str, str]], int] = _run_command) -> int: + return run( + [sys.executable, "-m", "prisma", "generate", "--schema", str(SCHEMA)], + REPO_ROOT, + env_with_own_bin_first(os.environ), + ) + + def main() -> int: version = importlib.metadata.version("prisma") expected = stamp_value(SCHEMA.read_bytes(), version) @@ -55,12 +82,9 @@ def main() -> int: f"(prisma {version}); skipping prisma generate" ) return 0 - result = subprocess.run( - [sys.executable, "-m", "prisma", "generate", "--schema", str(SCHEMA)], - cwd=REPO_ROOT, - ) - if result.returncode != 0: - return result.returncode + returncode = run_generate() + if returncode != 0: + return returncode STAMP.write_text(expected) return 0 diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index f9d7f2912fd..2fab7f8363a 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -34,8 +34,8 @@ skipped outright. When it is needed, it is a second basedpyright pass over a detached worktree at the merge-base, run under the same environment so import resolution matches, and its per-rule counts are cached under the repo's git common dir keyed by merge-base commit, ``pyrightconfig.json``, ``uv.lock``, -and the dependency-group set, so re-runs against the same branch -point pay for it once. A CI workflow publishes every staging commit's counts as +the Prisma schema, and the dependency-group set, so re-runs against the same +branch point pay for it once. A CI workflow publishes every staging commit's counts as an artifact (``--emit-counts-dir`` is its entry point), and on a disk-cache miss the gate first tries to download the merge-base's artifact through the ``gh`` CLI; any fetch failure falls back silently to the local base pass, so the gate @@ -83,6 +83,7 @@ GH_TIMEOUT_SECONDS = 10 TYPECHECK_ENV_DIR = REPO_ROOT / ".venv-typecheck" TYPECHECK_DEP_GROUPS = ("proxy-dev", "e2e-dev") PRISMA_GENERATE_SCRIPT = REPO_ROOT / "scripts" / "prisma_generate_if_needed.py" +PRISMA_SCHEMA = REPO_ROOT / "litellm" / "proxy" / "schema.prisma" # basedpyright's node process needs more than the ~4 GB default heap on this # repo; appended last so it wins node's last-flag-wins resolution over any @@ -192,6 +193,11 @@ def ensure_typecheck_env( measurement pass. Unconditional on purpose: an up-to-date env makes both steps near-instant no-ops, and skipping them on a heuristic is how the measured environment and the fingerprinted one drift apart.""" + if not env_dir.exists(): + sys.stderr.write( + f"provisioning {env_dir.name} (first run installs packages and " + "generates the Prisma client; re-runs are near-instant no-ops)\n" + ) env: Final = {**os.environ, "UV_PROJECT_ENVIRONMENT": str(env_dir)} for cmd in typecheck_env_commands(env_dir): if run(cmd, env) != 0: @@ -298,7 +304,7 @@ def environment_fingerprints( return ( *( hashlib.sha256(path.read_bytes()).hexdigest() - for path in (PYRIGHT_CONFIG, UV_LOCK) + for path in (PYRIGHT_CONFIG, UV_LOCK, PRISMA_SCHEMA) if path.exists() ), "groups:" + ",".join(dep_groups), diff --git a/tests/test_litellm/test_prisma_generate_if_needed.py b/tests/test_litellm/test_prisma_generate_if_needed.py index 39b9fcc4202..c36f580e24f 100644 --- a/tests/test_litellm/test_prisma_generate_if_needed.py +++ b/tests/test_litellm/test_prisma_generate_if_needed.py @@ -1,4 +1,6 @@ import importlib.util +import os +import sys from pathlib import Path _MODULE_PATH = ( @@ -33,3 +35,30 @@ def test_skip_requires_a_generated_client_even_with_a_matching_stamp(tmp_path): expected = mod.stamp_value(b"schema", "0.11.0") stamp.write_text(expected) assert mod.should_skip(stamp, expected, client_generated=False) is False + + +def test_env_puts_this_interpreters_bin_dir_first_on_path(): + env = mod.env_with_own_bin_first({"PATH": "/usr/bin", "HOME": "/home"}) + bin_dir = str(Path(sys.executable).parent) + assert env["PATH"].split(os.pathsep) == [bin_dir, "/usr/bin"] + assert env["HOME"] == "/home" + + +def test_env_without_an_inherited_path_is_just_the_bin_dir(): + env = mod.env_with_own_bin_first({}) + assert env["PATH"] == str(Path(sys.executable).parent) + + +def test_generate_runs_prisma_with_its_own_bin_dir_leading_the_childs_path(): + seen = {} + + def recorder(cmd, cwd, env): + seen["cmd"] = cmd + seen["cwd"] = cwd + seen["env"] = env + return 0 + + assert mod.run_generate(run=recorder) == 0 + assert seen["cmd"][:4] == [sys.executable, "-m", "prisma", "generate"] + assert seen["cwd"] == mod.REPO_ROOT + assert seen["env"]["PATH"].split(os.pathsep)[0] == str(Path(sys.executable).parent) diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py index 2d9731db53f..6cf21707629 100644 --- a/tests/test_litellm/test_type_check_gate.py +++ b/tests/test_litellm/test_type_check_gate.py @@ -1,3 +1,4 @@ +import hashlib import importlib.util import json import subprocess @@ -278,6 +279,11 @@ def test_fingerprints_carry_the_dependency_group_set(): ) +def test_fingerprints_cover_the_prisma_schema(): + schema_hash = hashlib.sha256(gate.PRISMA_SCHEMA.read_bytes()).hexdigest() + assert schema_hash in gate.environment_fingerprints() + + def test_env_commands_sync_the_canonical_groups_then_generate_prisma(): sync, generate = gate.typecheck_env_commands(Path("/envdir")) assert sync[:3] == ("uv", "sync", "--frozen") @@ -327,6 +333,22 @@ def test_ensure_env_fails_loudly_and_stops_at_the_first_failed_step(tmp_path): assert len(calls) == 1 +def test_ensure_env_announces_a_cold_provision(tmp_path, capsys): + def runner(cmd, env): + return 0 + + gate.ensure_typecheck_env(env_dir=tmp_path / "fresh", run=runner) + assert "provisioning" in capsys.readouterr().err + + +def test_ensure_env_is_silent_when_the_env_already_exists(tmp_path, capsys): + def runner(cmd, env): + return 0 + + gate.ensure_typecheck_env(env_dir=tmp_path, run=runner) + assert capsys.readouterr().err == "" + + def test_cached_counts_round_trip(tmp_path): path = gate.cache_path(tmp_path, "abc123", ("f1", "f2")) gate.store_counts(tmp_path, path, "abc123", {"reportAny": 3, "reportCall": 1}) From 14d4897e55a8223ee3b2815b0b4e038caa3c0f61 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 01:58:06 -0700 Subject: [PATCH 063/672] fix(guardrails): refuse scan_only_tool_results combos that scan nothing Prompt Security drops tool and function rows unless check_tool_results is on, so it now reports scan-only support from that setting and the registry refuses the pairing at boot. Pairing scan_only_tool_results with skip_tool_message_in_guardrail excludes every message, so guardrail initialization now rejects that combination too. --- .../prompt_security/prompt_security.py | 3 ++ .../proxy/guardrails/guardrail_registry.py | 15 +++++-- .../guardrails/test_guardrail_registry.py | 42 +++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 743ad888949..1a2c46f306c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -74,6 +74,9 @@ class PromptSecurityGuardrail(CustomGuardrail): super().__init__(**kwargs) + def supports_scan_only_tool_results(self) -> bool: + return self.check_tool_results + @log_guardrail_information async def apply_guardrail( self, diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 15e884c939c..9f70ed63dcb 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -16,6 +16,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, + effective_skip_tool_message_for_guardrail, ) from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrail, @@ -496,15 +497,21 @@ class InMemoryGuardrailHandler: "scan_only_tool_results", ): setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) - if ( - effective_scan_only_tool_results_for_guardrail(custom_guardrail_callback) - and not custom_guardrail_callback.supports_scan_only_tool_results() - ): + scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail( + custom_guardrail_callback + ) + if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): raise ValueError( f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results is enabled, but this " "guardrail's role filtering never scans tool results, so no request content would ever " "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." ) + if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): + raise ValueError( + f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results and " + "skip_tool_message_in_guardrail are enabled together, which excludes every message from " + "scanning, so no request content would ever be scanned. Remove one of the two." + ) configured_run_in_parallel: Final = getattr(litellm_params, "run_in_parallel", None) if configured_run_in_parallel is not None: custom_guardrail_callback.run_in_parallel = bool(configured_run_in_parallel) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 3053664ef27..729dbce6b9a 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -615,3 +615,45 @@ class TestScanOnlyToolResultsInitRefusal: }, ) assert result is not None + + def test_prompt_security_default_tool_filtering_rejects_scan_only_tool_results(self, monkeypatch): + monkeypatch.delenv("PROMPT_SECURITY_CHECK_TOOL_RESULTS", raising=False) + with pytest.raises(ValueError, match="never scans tool results"): + self._initialize( + "prompt-security-scan-only-combo", + { + "guardrail": "prompt_security", + "mode": "pre_call", + "api_key": "test-key", + "api_base": "https://ps.example.com", + "scan_only_tool_results": True, + }, + ) + + def test_prompt_security_check_tool_results_accepts_scan_only_tool_results(self, monkeypatch): + monkeypatch.setenv("PROMPT_SECURITY_CHECK_TOOL_RESULTS", "true") + result = self._initialize( + "prompt-security-scan-only-ok", + { + "guardrail": "prompt_security", + "mode": "pre_call", + "api_key": "test-key", + "api_base": "https://ps.example.com", + "scan_only_tool_results": True, + }, + ) + assert result is not None + + def test_skip_tool_message_with_scan_only_tool_results_is_rejected(self): + with pytest.raises(ValueError, match="skip_tool_message_in_guardrail are enabled together"): + self._initialize( + "bedrock-skip-tool-scan-only-combo", + { + "guardrail": "bedrock", + "mode": "pre_call", + "guardrailIdentifier": "gr-1", + "guardrailVersion": "1", + "skip_tool_message_in_guardrail": True, + "scan_only_tool_results": True, + }, + ) From 59041240f036fe80776b297b36757c48d85f7978 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 02:23:32 -0700 Subject: [PATCH 064/672] fix(managed_files): cap batch list page size at 100 and bulk-resolve raw file ids in one query --- .../proxy/hooks/managed_files.py | 104 +++++++++++++----- .../openai_files_endpoints/common_utils.py | 30 +++++ .../proxy/hooks/test_managed_files.py | 94 +++++++++++----- 3 files changed, 172 insertions(+), 56 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 07a1f959940..6fa6ef46ad4 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -3,6 +3,7 @@ import base64 import json +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Union, cast from uuid import NAMESPACE_URL, uuid5 @@ -31,10 +32,12 @@ from litellm.proxy._types import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + apply_unified_file_ids, ensure_batch_response_managed_file_ids, get_batch_id_from_unified_batch_id, get_content_type_from_file_object, get_model_id_from_unified_batch_id, + map_raw_file_ids_to_unified, normalize_mime_type_for_provider, resolve_managed_output_file_model_name, ) @@ -62,6 +65,9 @@ if TYPE_CHECKING: if TYPE_CHECKING: from opentelemetry.trace import Span as _Span + from prisma.models import ( + LiteLLM_ManagedObjectTable as PrismaManagedObjectRow, + ) from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache from litellm.proxy.utils import PrismaClient as _PrismaClient @@ -75,6 +81,20 @@ else: PrismaClient = Any +def _decode_json_blob(blob: object) -> object: + return json.loads(blob) if isinstance(blob, str) else blob + + +def _parse_managed_batch_row(row: "PrismaManagedObjectRow") -> Optional[LiteLLMBatch]: + try: + batch_obj: Final = LiteLLMBatch.model_validate(_decode_json_blob(row.file_object)) + except Exception as e: + verbose_logger.warning(f"Failed to parse batch object {row.unified_object_id}: {e}") + return None + batch_obj.id = row.unified_object_id + return batch_obj + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -329,7 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): detail=f"Invalid 'after' cursor: no batch found with id '{after}'.", ) - page_size = limit or 20 + page_size: Final = min(limit or 20, 100) cursor_args: Dict[str, Any] = ( {"cursor": {"unified_object_id": after}, "skip": 1} if after else {} ) @@ -343,36 +363,60 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): has_more = len(batches) > page_size - batch_objects: List[LiteLLMBatch] = [] - for batch in batches[:page_size]: - try: - batch_data = ( - json.loads(batch.file_object) - if isinstance(batch.file_object, str) - else batch.file_object - ) - batch_obj = LiteLLMBatch.model_validate(batch_data) - batch_obj.id = batch.unified_object_id - await ensure_batch_response_managed_file_ids( - response=batch_obj, - managed_files_obj=self, - prisma_client=self.prisma_client, - verbose_proxy_logger=verbose_logger, - user_api_key_dict=user_api_key_dict, - db_batch_object=batch, - unified_batch_id=_is_base64_encoded_unified_file_id( - batch.unified_object_id - ), - ) - batch_objects.append(batch_obj) + parsed_rows: Final = tuple( + (row, batch_obj) + for row in batches[:page_size] + if (batch_obj := _parse_managed_batch_row(row)) is not None + ) + unified_id_by_raw_id: Final = await map_raw_file_ids_to_unified( + raw_file_ids=frozenset( + file_id + for _, batch_obj in parsed_rows + for file_id in (batch_obj.input_file_id, batch_obj.output_file_id, batch_obj.error_file_id) + if file_id and not _is_base64_encoded_unified_file_id(file_id) + ), + prisma_client=self.prisma_client, + ) + resolved_batches: Final = [ + await self._resolve_listed_batch( + row=row, + batch_obj=batch_obj, + unified_id_by_raw_id=unified_id_by_raw_id, + user_api_key_dict=user_api_key_dict, + ) + for row, batch_obj in parsed_rows + ] + return build_list_page( + [batch_obj for batch_obj in resolved_batches if batch_obj is not None], + has_more=has_more, + ) - except Exception as e: - verbose_logger.warning( - f"Failed to parse batch object {batch.unified_object_id}: {e}" - ) - continue - - return build_list_page(batch_objects, has_more=has_more) + async def _resolve_listed_batch( + self, + row: "PrismaManagedObjectRow", + batch_obj: LiteLLMBatch, + unified_id_by_raw_id: Mapping[str, str], + user_api_key_dict: UserAPIKeyAuth, + ) -> Optional[LiteLLMBatch]: + apply_unified_file_ids(batch_obj, unified_id_by_raw_id) + try: + await ensure_batch_response_managed_file_ids( + response=batch_obj, + managed_files_obj=self, + prisma_client=self.prisma_client, + verbose_proxy_logger=verbose_logger, + user_api_key_dict=user_api_key_dict, + db_batch_object=row, + unified_batch_id=_is_base64_encoded_unified_file_id( + row.unified_object_id + ), + ) + except Exception as e: + verbose_logger.warning( + f"Failed to resolve managed file ids for batch {row.unified_object_id}: {e}" + ) + return None + return batch_obj async def get_user_created_file_ids( self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str] diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 080b8b80ae4..bf83a7cf25c 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1,6 +1,7 @@ import base64 import mimetypes import re +from collections.abc import Mapping from dataclasses import dataclass, field from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Optional @@ -16,6 +17,7 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_ManagedObjectTable from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import PrismaClient from litellm.router import Router from litellm.types.utils import LiteLLMBatch @@ -1002,6 +1004,34 @@ async def resolve_output_file_ids_to_unified(response, prisma_client) -> None: pass +async def map_raw_file_ids_to_unified( + raw_file_ids: frozenset[str], prisma_client: "PrismaClient | None" +) -> Mapping[str, str]: + if not raw_file_ids or not prisma_client: + return MappingProxyType({}) + managed_files: Final = await ManagedFileRepository(prisma_client).table.find_many( + where={"flat_model_file_ids": {"hasSome": sorted(raw_file_ids)}} # mutable-ok: prisma where is a plain dict + ) + return MappingProxyType( + { + raw_id: managed_file.unified_file_id + for managed_file in managed_files + for raw_id in managed_file.flat_model_file_ids + if raw_id in raw_file_ids + } + ) + + +def apply_unified_file_ids(response: "LiteLLMBatch", unified_id_by_raw_id: Mapping[str, str]) -> None: + for file_attr, raw_id in ( + ("input_file_id", getattr(response, "input_file_id", None)), + ("output_file_id", getattr(response, "output_file_id", None)), + ("error_file_id", getattr(response, "error_file_id", None)), + ): + if isinstance(raw_id, str) and raw_id in unified_id_by_raw_id: + setattr(response, file_attr, unified_id_by_raw_id[raw_id]) + + async def ensure_batch_response_managed_file_ids( response, managed_files_obj, diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index fc10a1257e1..e1e5cc6c532 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -1869,15 +1869,12 @@ async def test_list_batches_registers_and_returns_unified_output_file_ids(): input_file_row = MagicMock() input_file_row.unified_file_id = unified_input_file_id + input_file_row.flat_model_file_ids = [raw_input_file_id] - def find_managed_file(where): - if where["flat_model_file_ids"]["has"] == raw_input_file_id: - return input_file_row - return None - - prisma_client.db.litellm_managedfiletable.find_first = AsyncMock( - side_effect=find_managed_file + prisma_client.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=[input_file_row] ) + prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) proxy_managed_files = _PROXY_LiteLLMManagedFiles( DualCache(), prisma_client=prisma_client @@ -1892,6 +1889,13 @@ async def test_list_batches_registers_and_returns_unified_output_file_ids(): assert listed.id == unified_batch_uid assert listed.input_file_id == unified_input_file_id + bulk_lookup = prisma_client.db.litellm_managedfiletable.find_many.await_args + assert set(bulk_lookup.kwargs["where"]["flat_model_file_ids"]["hasSome"]) == { + raw_input_file_id, + raw_output_file_id, + raw_error_file_id, + } + decoded_output = _decode_unified_id(listed.output_file_id) assert decoded_output.startswith("litellm_proxy") assert f"llm_output_file_id,{raw_output_file_id}" in decoded_output @@ -1914,33 +1918,43 @@ async def test_list_batches_registers_and_returns_unified_output_file_ids(): @pytest.mark.asyncio async def test_list_batches_resolves_existing_managed_rows_without_minting(): """When the raw provider file IDs already have managed file rows, listing must - swap in the existing unified IDs and must not upsert duplicate rows.""" + swap in the existing unified IDs via one bulk lookup for the whole page, with + no per-row queries and no duplicate upserts.""" from litellm.proxy._types import UserAPIKeyAuth - unified_batch_uid = _create_unified_batch_id("model-123", "batch-456") - raw_output_file_id = "file-list-out-existing" - existing_unified_output_id = base64.urlsafe_b64encode( - f"litellm_proxy:application/json;unified_id,u-9;llm_output_file_id,{raw_output_file_id}".encode() + unified_input_file_id = base64.urlsafe_b64encode( + b"litellm_proxy:application/octet-stream;unified_id,in-9;target_model_names,gpt-5-batch" ).decode() + raw_output_file_ids = ["file-list-out-existing-1", "file-list-out-existing-2"] + existing_unified_output_ids = [ + base64.urlsafe_b64encode( + f"litellm_proxy:application/json;unified_id,u-{i};llm_output_file_id,{raw_id}".encode() + ).decode() + for i, raw_id in enumerate(raw_output_file_ids) + ] - record = _terminal_batch_record( - unified_batch_uid, "file-list-in-9", raw_output_file_id, "" - ) + records = [ + _terminal_batch_record( + _create_unified_batch_id("model-123", f"batch-{i}"), + unified_input_file_id, + raw_id, + "", + ) + for i, raw_id in enumerate(raw_output_file_ids) + ] prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_many.return_value = [record] + prisma_client.db.litellm_managedobjecttable.find_many.return_value = records - existing_row = MagicMock() - existing_row.unified_file_id = existing_unified_output_id + existing_rows = [ + MagicMock(unified_file_id=unified_id, flat_model_file_ids=[raw_id]) + for raw_id, unified_id in zip(raw_output_file_ids, existing_unified_output_ids) + ] - def find_managed_file(where): - if where["flat_model_file_ids"]["has"] == raw_output_file_id: - return existing_row - return None - - prisma_client.db.litellm_managedfiletable.find_first = AsyncMock( - side_effect=find_managed_file + prisma_client.db.litellm_managedfiletable.find_many = AsyncMock( + return_value=existing_rows ) + prisma_client.db.litellm_managedfiletable.find_first = AsyncMock() proxy_managed_files = _PROXY_LiteLLMManagedFiles( DualCache(), prisma_client=prisma_client @@ -1951,10 +1965,38 @@ async def test_list_batches_resolves_existing_managed_rows_without_minting(): limit=10, ) - assert result["data"][0].output_file_id == existing_unified_output_id + assert [b.output_file_id for b in result["data"]] == existing_unified_output_ids + prisma_client.db.litellm_managedfiletable.find_many.assert_awaited_once() + prisma_client.db.litellm_managedfiletable.find_first.assert_not_awaited() prisma_client.db.litellm_managedfiletable.upsert.assert_not_awaited() +@pytest.mark.asyncio +async def test_list_batches_caps_page_size_at_100(): + """The list page size must be capped at 100 rows (matching OpenAI's limit) + even when the caller asks for more, so one request cannot fan out into an + unbounded scan.""" + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_many.return_value = [] + + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.list_user_batches( + user_api_key_dict=UserAPIKeyAuth(user_id="owner-user"), + limit=100000, + ) + + assert ( + prisma_client.db.litellm_managedobjecttable.find_many.await_args.kwargs["take"] + == 101 + ) + assert result["data"] == [] + + @pytest.mark.asyncio async def test_list_batches_from_managed_objects_table_provider_filter_raises_exception(): from litellm.proxy._types import UserAPIKeyAuth From 526f6d793ec96d9ba1ce93a854f6486499bff0e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 02:23:52 -0700 Subject: [PATCH 065/672] fix(lint): retire the single-slot base-counts cache Storing a baseline used to prune every other cache entry, so gate runs in concurrent worktrees kept evicting each other's baselines and forcing full recomputes: this bit six times across two nights of benchmarking. The store now writes alongside existing entries and evicts only the oldest beyond eight, keyed as before by merge-base and environment fingerprint, so parallel worktrees' baselines simply coexist --- scripts/type_check_gate.py | 23 +++++++++++--- tests/test_litellm/test_type_check_gate.py | 35 ++++++++++++++++++++-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index 2fab7f8363a..c9f774c6113 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -63,7 +63,7 @@ import sys import tempfile import zipfile from collections import Counter -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from pathlib import Path from typing import Final, NamedTuple @@ -73,6 +73,7 @@ PYRIGHT_CONFIG = REPO_ROOT / "pyrightconfig.json" UV_LOCK = REPO_ROOT / "uv.lock" DEFAULT_BASE = "origin/litellm_internal_staging" CACHE_FILE_PREFIX = "basedpyright-base-" +CACHE_KEEP_ENTRIES = 8 ARTIFACT_NAME_PREFIX = "basedpyright-counts-" GH_TIMEOUT_SECONDS = 10 @@ -367,16 +368,30 @@ def counts_payload(base_point: str, counts: Mapping[str, int]) -> str: ) +def entry_recency(path: Path) -> float: + try: + return path.stat().st_mtime + except OSError: + return 0.0 + + +def evicted_beyond_cap(entries: Sequence[Path], keep: int) -> tuple[Path, ...]: + newest_first: Final = sorted(entries, key=entry_recency, reverse=True) + return tuple(newest_first[keep:]) + + def store_counts( directory: Path, path: Path, base_point: str, counts: Mapping[str, int] ) -> None: directory.mkdir(parents=True, exist_ok=True) - for stale in directory.glob(f"{CACHE_FILE_PREFIX}*.json"): - if stale != path: - stale.unlink(missing_ok=True) scratch = scratch_path(path) scratch.write_text(counts_payload(base_point, counts)) scratch.replace(path) + siblings: Final = tuple( + entry for entry in directory.glob(f"{CACHE_FILE_PREFIX}*.json") if entry != path + ) + for stale in evicted_beyond_cap(siblings, CACHE_KEEP_ENTRIES - 1): + stale.unlink(missing_ok=True) def parse_origin_slug(url: str) -> str | None: diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py index 6cf21707629..d104e4ca0c8 100644 --- a/tests/test_litellm/test_type_check_gate.py +++ b/tests/test_litellm/test_type_check_gate.py @@ -1,6 +1,7 @@ import hashlib import importlib.util import json +import os import subprocess from pathlib import Path @@ -387,15 +388,45 @@ def test_store_prune_spares_a_concurrent_runs_in_flight_scratch(tmp_path): assert gate.load_cached_counts(mine) == {"reportAny": 1} -def test_store_prunes_entries_for_other_branch_points(tmp_path): +def test_store_keeps_a_concurrent_worktrees_entry_for_another_branch_point(tmp_path): old = gate.cache_path(tmp_path, "old", ("f",)) gate.store_counts(tmp_path, old, "old", {"reportAny": 1}) new = gate.cache_path(tmp_path, "new", ("f",)) gate.store_counts(tmp_path, new, "new", {"reportAny": 2}) - assert not old.exists() + assert gate.load_cached_counts(old) == {"reportAny": 1} assert gate.load_cached_counts(new) == {"reportAny": 2} +def test_store_evicts_only_the_oldest_entries_beyond_the_cap(tmp_path): + aged = [ + gate.cache_path(tmp_path, f"base{i}", ("f",)) + for i in range(gate.CACHE_KEEP_ENTRIES) + ] + for age, path in enumerate(aged): + gate.store_counts(tmp_path, path, f"base{age}", {"reportAny": age}) + os.utime(path, (age, age)) + newest = gate.cache_path(tmp_path, "newest", ("f",)) + gate.store_counts(tmp_path, newest, "newest", {"reportAny": 99}) + assert not aged[0].exists() + assert all(path.exists() for path in aged[1:]) + assert gate.load_cached_counts(newest) == {"reportAny": 99} + + +def test_store_never_evicts_the_entry_it_just_wrote_even_on_mtime_ties(tmp_path): + others = [ + gate.cache_path(tmp_path, f"base{i}", ("f",)) + for i in range(gate.CACHE_KEEP_ENTRIES + 2) + ] + for path in others: + gate.store_counts(tmp_path, path, path.name, {"reportAny": 1}) + os.utime(path, (9_999_999_999, 9_999_999_999)) + mine = gate.cache_path(tmp_path, "mine", ("f",)) + gate.store_counts(tmp_path, mine, "mine", {"reportAny": 2}) + assert gate.load_cached_counts(mine) == {"reportAny": 2} + survivors = list(tmp_path.glob(f"{gate.CACHE_FILE_PREFIX}*.json")) + assert len(survivors) == gate.CACHE_KEEP_ENTRIES + + def _no_fetch(ref): return None From 470ebc208991c0883386bba4a7bd4b9c1d7d2c09 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 02:27:59 -0700 Subject: [PATCH 066/672] test(batches): cover caller-supplied db row skipping the cancel-path lookup --- ..._batch_update_db_managed_output_file_id.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py index 74139fa9238..1b60c97b510 100644 --- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py +++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py @@ -127,6 +127,42 @@ async def test_cancel_path_registers_output_file_under_batch_owner(): assert stored["output_file_id"] == unified_id +@pytest.mark.asyncio +async def test_update_batch_skips_lookup_when_db_batch_object_supplied(): + unified_id = "file-bWFuYWdlZF9vdXRwdXRfaWQ=" + caller_row = SimpleNamespace( + created_by="caller-owner", team_id="caller-team", status="in_progress" + ) + decoy_row = SimpleNamespace( + created_by="decoy-owner", team_id="decoy-team", status="in_progress" + ) + response = _build_batch_response( + status="cancelling", + output_file_id="file-raw-output", + hidden_params={"model_id": "my-model", "model_name": "openai/gpt-4o"}, + ) + mock_managed_files = _build_managed_files_mock(unified_id=unified_id) + mock_prisma = _build_prisma_mock(db_batch_object=decoy_row) + + await update_batch_in_database( + batch_id="batch_managed_ids_test", + unified_batch_id="litellm_proxy;model_id:my-model;llm_batch_id:batch_managed_ids_test", + response=response, + managed_files_obj=mock_managed_files, + prisma_client=mock_prisma, + verbose_proxy_logger=MagicMock(), + db_batch_object=caller_row, + operation="retrieve", + ) + + mock_prisma.db.litellm_managedobjecttable.find_first.assert_not_called() + forwarded_auth = mock_managed_files.store_unified_file_id.call_args.kwargs[ + "user_api_key_dict" + ] + assert forwarded_auth.user_id == "caller-owner" + assert forwarded_auth.team_id == "caller-team" + + @pytest.mark.asyncio async def test_update_batch_derives_model_id_from_unified_batch_id(): unified_id = "file-bWFuYWdlZF9vdXRwdXRfaWQ=" From 920e05c484337350af5724d46b7233d4ab8eaacc Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 02:45:44 -0700 Subject: [PATCH 067/672] chore: fix typo --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4d66297e0a0..fa891bc7ae9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,7 +59,7 @@ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages When working on a PR, keep the PR description in sync with new commits being made -All GitHub comments must be human-readable and 15-25 word max +All GitHub comments must be human-readable and 15-25 words max Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in From 0c1a4b127dbfdf8aca292e2344f6c76b678d7d75 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:31:40 -0700 Subject: [PATCH 068/672] chore(lint): name MappingProxyType in the mutable-collection fix messages LIT001/LIT002 and the typing.Dict ban all steered dict-shaped values to frozen dataclasses or suppression even though the checker already accepts MappingProxyType as a freezing wrapper; the messages now name it so the dict-shaped freeze path is actually discoverable at fix time. --- CLAUDE.md | 4 ++-- ruff-strict.toml | 2 +- scripts/check_type_discipline.py | 16 ++++++++++------ tests/test_litellm/test_check_type_discipline.py | 8 ++++++++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1bc4d108da7..81e560af849 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-bud If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in -If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason +If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()`, and freeze dict-shaped values with `types.MappingProxyType({...})` (annotated as `Mapping[...]`), instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # `. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing @@ -72,7 +72,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - Composition over inheritance - Never-nester: early returns over deep nesting - Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never) -- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), etc. +- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `types.MappingProxyType`, etc. - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: ` explaining why - Use dependency injection - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed diff --git a/ruff-strict.toml b/ruff-strict.toml index d58885fe848..66d8f281fce 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -15,7 +15,7 @@ max-args = 5 "typing.Any".msg = "Use a concrete type. Frozen slots=True dataclass (preferred) / NamedTuple / ReadOnly TypedDict for payloads." "typing_extensions.Any".msg = "Same as typing.Any." "typing.List".msg = "tuple[X, ...] for state, Sequence[X] for params." -"typing.Dict".msg = "Frozen dataclass / NamedTuple / ReadOnly TypedDict; create a Mapping alias with concrete value types if truly dynamic." +"typing.Dict".msg = "Frozen dataclass / NamedTuple / ReadOnly TypedDict; if truly dynamic, a Mapping alias with concrete value types, frozen at runtime with types.MappingProxyType({...})." "typing.Set".msg = "frozenset[X] or AbstractSet[X]." "typing.MutableSequence".msg = "Sequence[X]." "typing.MutableMapping".msg = "See typing.Dict." diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index c303fbaffce..d73f1095895 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -11,14 +11,16 @@ LIT001 Mutable collection in a type annotation, anywhere it appears: function collection lets whoever holds it grow or rewrite it after the fact; annotate a read-only view instead (Mapping/Sequence/AbstractSet/tuple[X, ...]/ frozenset[X], or a frozen dataclass / NamedTuple / ReadOnly TypedDict) and - build it functionally (comprehension / map, not append-in-a-loop). + build it functionally (comprehension / map / MappingProxyType({...}), not + append-in-a-loop). Suppress with `# mutable-ok: ` on the offending line. LIT002 Mutable-collection *construction*: a list/dict/set literal or comprehension, or a call to a mutable constructor (list/dict/set/deque/defaultdict/Counter/...). Catches the unannotated seed-then-mutate pattern LIT001 cannot see (`acc = []`). Build the value in one shot and freeze it: a `tuple`/`frozenset` wrapping a - generator (`tuple(f(x) for x in xs)`), a tuple literal, or a frozen dataclass / - NamedTuple / ReadOnly TypedDict. Generator expressions and `tuple`/`frozenset` + generator (`tuple(f(x) for x in xs)`), a tuple literal, `MappingProxyType({...})` + for a dict-shaped value, or a frozen dataclass / NamedTuple / ReadOnly TypedDict. + Generator expressions and `tuple`/`frozenset` calls are not construction and pass. Annotation-internal lists (`Callable[[int], str]`) are exempt, as is a value passed directly to a freezing wrapper (`tuple(...)`, `frozenset(...)`, `MappingProxyType(...)`): it is frozen before @@ -279,7 +281,8 @@ def _mutable_ann(path: Path, line: int, name: str, where: str) -> Violation: f"mutable `{name}` in {where}: a mutable collection can be grown or rewritten " f"by whoever holds it. Annotate a read-only view -- Mapping[...], Sequence[...], " f"AbstractSet[...], tuple[X, ...], frozenset[X], or a frozen dataclass / " - f"NamedTuple / ReadOnly TypedDict -- and build it functionally, not by " + f"NamedTuple / ReadOnly TypedDict -- and build it functionally " + f"(comprehension / map / `MappingProxyType({{...}})`), not by " f"append-in-a-loop (suppress: `# mutable-ok: `)", ) @@ -488,8 +491,9 @@ def iter_construction_violations(path: Path, tree: ast.AST, comments: Comments) path, node.lineno, "LIT002", f"mutable {kind}: this builds a collection that can be grown or rewritten. " f"Build it in one shot and freeze it -- a tuple/frozenset wrapping a generator " - f"(`tuple(f(x) for x in xs)`), a tuple literal, or a frozen dataclass / NamedTuple " - f"/ ReadOnly TypedDict (suppress: `# mutable-ok: `)", + f"(`tuple(f(x) for x in xs)`), a tuple literal, `MappingProxyType({{...}})` for a " + f"dict-shaped value, or a frozen dataclass / NamedTuple / ReadOnly TypedDict " + f"(suppress: `# mutable-ok: `)", ) diff --git a/tests/test_litellm/test_check_type_discipline.py b/tests/test_litellm/test_check_type_discipline.py index f2bfc637095..81efbf0a4f8 100644 --- a/tests/test_litellm/test_check_type_discipline.py +++ b/tests/test_litellm/test_check_type_discipline.py @@ -175,6 +175,14 @@ def test_unfrozen_literal_still_counts(tmp_path): assert "LIT002" in _codes(tmp_path, "from types import MappingProxyType\nd = {'a': 1}\nm = MappingProxyType(d)\n") +def test_fix_messages_name_mappingproxytype(tmp_path): + f = tmp_path / "snippet.py" + f.write_text("x: dict[str, int] = {}\n", encoding="utf-8") + messages = {v.code: v.message for v in checker.check_file(f)} + assert "MappingProxyType" in messages["LIT001"] + assert "MappingProxyType" in messages["LIT002"] + + def test_mutable_ok_with_reason_suppresses_both_rules(tmp_path): codes = _codes(tmp_path, "x: dict[str, int] = {} # mutable-ok: in-place buffer mutated hot path\n") assert "LIT001" not in codes From d7ca4dc77f04e1dbd903fd15de26e508504ba64f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:34:34 -0700 Subject: [PATCH 069/672] chore(lint): say annotate Mapping[...] instead of Mapping alias in typing.Dict ban --- ruff-strict.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff-strict.toml b/ruff-strict.toml index 66d8f281fce..5e5192aa864 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -15,7 +15,7 @@ max-args = 5 "typing.Any".msg = "Use a concrete type. Frozen slots=True dataclass (preferred) / NamedTuple / ReadOnly TypedDict for payloads." "typing_extensions.Any".msg = "Same as typing.Any." "typing.List".msg = "tuple[X, ...] for state, Sequence[X] for params." -"typing.Dict".msg = "Frozen dataclass / NamedTuple / ReadOnly TypedDict; if truly dynamic, a Mapping alias with concrete value types, frozen at runtime with types.MappingProxyType({...})." +"typing.Dict".msg = "Frozen dataclass / NamedTuple / ReadOnly TypedDict; if truly dynamic, annotate Mapping[...] with concrete value types and freeze at runtime with types.MappingProxyType({...})." "typing.Set".msg = "frozenset[X] or AbstractSet[X]." "typing.MutableSequence".msg = "Sequence[X]." "typing.MutableMapping".msg = "See typing.Dict." From d0712e1a5e8b116f603fad6160b3bee298863ecc Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:37:46 -0700 Subject: [PATCH 070/672] chore: make CLAUDE.md more concise --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 81e560af849..a66e36e75cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-bud If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in -If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()`, and freeze dict-shaped values with `types.MappingProxyType({...})` (annotated as `Mapping[...]`), instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason +If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # `. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing @@ -72,7 +72,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - Composition over inheritance - Never-nester: early returns over deep nesting - Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never) -- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `types.MappingProxyType`, etc. +- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc. - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: ` explaining why - Use dependency injection - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed From 818e319b536fb986294ca5ae8ab75389569137e1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:38:28 -0700 Subject: [PATCH 071/672] chore: make it more concise --- ruff-strict.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff-strict.toml b/ruff-strict.toml index 5e5192aa864..01faf04805f 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -15,7 +15,7 @@ max-args = 5 "typing.Any".msg = "Use a concrete type. Frozen slots=True dataclass (preferred) / NamedTuple / ReadOnly TypedDict for payloads." "typing_extensions.Any".msg = "Same as typing.Any." "typing.List".msg = "tuple[X, ...] for state, Sequence[X] for params." -"typing.Dict".msg = "Frozen dataclass / NamedTuple / ReadOnly TypedDict; if truly dynamic, annotate Mapping[...] with concrete value types and freeze at runtime with types.MappingProxyType({...})." +"typing.Dict".msg = "Frozen dataclass / NamedTuple / ReadOnly TypedDict; if truly dynamic, use MappingProxyType." "typing.Set".msg = "frozenset[X] or AbstractSet[X]." "typing.MutableSequence".msg = "Sequence[X]." "typing.MutableMapping".msg = "See typing.Dict." From 20a94a91001aa7f1ff01de3b8dc3774d4db3bbae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:57:44 -0700 Subject: [PATCH 072/672] chore(lint): move MappingProxyType to the dynamic tail of the LIT002 freeze menu Revert the LIT001 build-clause inserts, phrase the LIT002 menu as 'or (if it really must be dynamic) a MappingProxyType wrapping a dict literal or comprehension', and fold the two freezing-wrapper exemption sentences into one that names MappingProxyType beside tuple/frozenset. --- scripts/check_type_discipline.py | 28 +++++++++---------- .../test_check_type_discipline.py | 9 +++--- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index d73f1095895..65d0424fb5a 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -11,21 +11,20 @@ LIT001 Mutable collection in a type annotation, anywhere it appears: function collection lets whoever holds it grow or rewrite it after the fact; annotate a read-only view instead (Mapping/Sequence/AbstractSet/tuple[X, ...]/ frozenset[X], or a frozen dataclass / NamedTuple / ReadOnly TypedDict) and - build it functionally (comprehension / map / MappingProxyType({...}), not - append-in-a-loop). + build it functionally (comprehension / map, not append-in-a-loop). Suppress with `# mutable-ok: ` on the offending line. LIT002 Mutable-collection *construction*: a list/dict/set literal or comprehension, or a call to a mutable constructor (list/dict/set/deque/defaultdict/Counter/...). Catches the unannotated seed-then-mutate pattern LIT001 cannot see (`acc = []`). Build the value in one shot and freeze it: a `tuple`/`frozenset` wrapping a - generator (`tuple(f(x) for x in xs)`), a tuple literal, `MappingProxyType({...})` - for a dict-shaped value, or a frozen dataclass / NamedTuple / ReadOnly TypedDict. - Generator expressions and `tuple`/`frozenset` - calls are not construction and pass. Annotation-internal lists (`Callable[[int], - str]`) are exempt, as is a value passed directly to a freezing wrapper - (`tuple(...)`, `frozenset(...)`, `MappingProxyType(...)`): it is frozen before - it can escape, though anything mutable nested inside it still counts. - Suppress with `# mutable-ok: `. + generator (`tuple(f(x) for x in xs)`), a tuple literal, a frozen dataclass / + NamedTuple / ReadOnly TypedDict, or (if it really must be dynamic) a + MappingProxyType wrapping a dict literal or comprehension. Generator expressions + and freezing-wrapper calls (`tuple(...)`, `frozenset(...)`, + `MappingProxyType(...)`) are not construction and pass, as does the value passed + directly to a wrapper: it is frozen before it can escape, though anything + mutable nested inside it still counts. Annotation-internal lists + (`Callable[[int], str]`) are exempt. Suppress with `# mutable-ok: `. LIT003 noqa suppression without rule codes or without a reason. Required shape: `# noqa: TID251 # ` LIT004 pyright/mypy ignore without bracketed codes or without a reason. @@ -281,8 +280,7 @@ def _mutable_ann(path: Path, line: int, name: str, where: str) -> Violation: f"mutable `{name}` in {where}: a mutable collection can be grown or rewritten " f"by whoever holds it. Annotate a read-only view -- Mapping[...], Sequence[...], " f"AbstractSet[...], tuple[X, ...], frozenset[X], or a frozen dataclass / " - f"NamedTuple / ReadOnly TypedDict -- and build it functionally " - f"(comprehension / map / `MappingProxyType({{...}})`), not by " + f"NamedTuple / ReadOnly TypedDict -- and build it functionally, not by " f"append-in-a-loop (suppress: `# mutable-ok: `)", ) @@ -491,9 +489,9 @@ def iter_construction_violations(path: Path, tree: ast.AST, comments: Comments) path, node.lineno, "LIT002", f"mutable {kind}: this builds a collection that can be grown or rewritten. " f"Build it in one shot and freeze it -- a tuple/frozenset wrapping a generator " - f"(`tuple(f(x) for x in xs)`), a tuple literal, `MappingProxyType({{...}})` for a " - f"dict-shaped value, or a frozen dataclass / NamedTuple / ReadOnly TypedDict " - f"(suppress: `# mutable-ok: `)", + f"(`tuple(f(x) for x in xs)`), a tuple literal, a frozen dataclass / NamedTuple " + f"/ ReadOnly TypedDict, or (if it really must be dynamic) a MappingProxyType " + f"wrapping a dict literal or comprehension (suppress: `# mutable-ok: `)", ) diff --git a/tests/test_litellm/test_check_type_discipline.py b/tests/test_litellm/test_check_type_discipline.py index 81efbf0a4f8..4b8533df604 100644 --- a/tests/test_litellm/test_check_type_discipline.py +++ b/tests/test_litellm/test_check_type_discipline.py @@ -175,12 +175,11 @@ def test_unfrozen_literal_still_counts(tmp_path): assert "LIT002" in _codes(tmp_path, "from types import MappingProxyType\nd = {'a': 1}\nm = MappingProxyType(d)\n") -def test_fix_messages_name_mappingproxytype(tmp_path): +def test_lit002_fix_message_names_mappingproxytype(tmp_path): f = tmp_path / "snippet.py" - f.write_text("x: dict[str, int] = {}\n", encoding="utf-8") - messages = {v.code: v.message for v in checker.check_file(f)} - assert "MappingProxyType" in messages["LIT001"] - assert "MappingProxyType" in messages["LIT002"] + f.write_text("x = {'a': 1}\n", encoding="utf-8") + messages = [v.message for v in checker.check_file(f) if v.code == "LIT002"] + assert "MappingProxyType" in messages[0] def test_mutable_ok_with_reason_suppresses_both_rules(tmp_path): From 2c91166d32d5253ae7434c796c2365598c18b8b5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:39:34 +0000 Subject: [PATCH 073/672] chore: ignore the mechanical lint and typing sweeps in git blame Seven wide-reaching but semantically neutral commits landed since the last entry, together rewriting roughly 162k lines across ~4,700 file touches. Blame on any line they reflowed points at the sweep instead of the commit that wrote the logic. They cover the safe ruff autofix pass, the collections.abc import move, the f-string !s cleanup, lazy log message construction, the LIT010 and LIT011 Final and frozen-parameter rollout, ruff coverage for litellm/types, and the inert type: ignore strip. Smaller ratchet commits are left out on purpose: each touches a few hundred lines at most, so listing them would grow the file faster than it buys back blame accuracy --- .git-blame-ignore-revs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 2527239b904..a0943cff53d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -17,3 +17,24 @@ # style: unify ruff format width on 120 (#31518) 48b5a5a0cc5a694a11219416ee0b6eb6e620e74e + +# refactor(imports): move collections.abc names out of typing (#35495) +397e8e4918777e4e60a7f5e88699e0a9a7dabb3d + +# refactor(lint): apply every safe ruff autofix and zero 28 strict-rule budgets (#35495) +b604e2b20c6db2099085a2f0e59b7e99e87eed6f + +# refactor(logging): drop redundant !s conversion flags from f-strings (#35546) +7b2d3440cba3160277470f7a0180098ae9b87864 + +# perf: build log messages lazily so filtered-out log records cost nothing (#35703) +c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd + +# feat(lint): enforce Final on locals and freeze function parameters (#35807) +2708620d6a599cc73c1950a942d26ac26a7ed3d4 + +# chore(lint): remove litellm/types from the ruff lint exclusion (#35926) +4e32a8bf6a1e1af1e04b67c759841ccef44b2235 + +# chore(lint): strip inert type: ignore comments and zero LIT009/LIT010/LIT011 headroom (#35928) +338e411103ad5d7003e97f34f04fa36bca542dbe From ca7453bc6922f707761ac212b0c5872f777cd3fd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:04:38 +0000 Subject: [PATCH 074/672] chore(lint): recompute budget ceilings after merging base The base branch ratcheted the same limits in 28a277e9, so the conflicting files were reset to base and the ratchet re-run against the new merge-base rather than resolved by hand. Each limit is now the base value minus this branch's own delta, so both ratchets survive: basedpyright -653 across 48 rules, strict ruff -80, LIT -85. --- basedpyright-code-budget.json | 16 ++++++++-------- ruff-strict-budget.json | 18 +++++++++--------- type-discipline-budget.json | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index d98e6c6c911..632743236c4 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29204 + "limit": 28842 }, "reportArgumentType": { "limit": 2634 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 9225 + "limit": 9103 }, "reportFunctionMemberAccess": { "limit": 7 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5850 + "limit": 5843 }, "reportMissingTypeArgument": { - "limit": 15833 + "limit": 15816 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45145 + "limit": 45110 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 39881 + "limit": 39838 }, "reportUnknownParameterType": { - "limit": 20258 + "limit": 20237 }, "reportUnknownVariableType": { - "limit": 31429 + "limit": 31383 }, "reportUnnecessaryCast": { "limit": 122 diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 65c98f6aab3..60356eda05b 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,30 +1,30 @@ { "ANN001": { - "limit": 3126 + "limit": 3121 }, "ANN002": { "limit": 71 }, "ANN003": { - "limit": 836 + "limit": 834 }, "ANN201": { - "limit": 2037 + "limit": 2033 }, "ANN202": { - "limit": 869 + "limit": 865 }, "ANN204": { - "limit": 715 + "limit": 713 }, "ANN205": { - "limit": 115 + "limit": 114 }, "ANN206": { "limit": 133 }, "ANN401": { - "limit": 1689 + "limit": 1630 }, "ASYNC230": { "limit": 11 @@ -222,7 +222,7 @@ "limit": 0 }, "RET504": { - "limit": 178 + "limit": 177 }, "RUF010": { "limit": 0 @@ -306,7 +306,7 @@ "limit": 0 }, "TID251": { - "limit": 1242 + "limit": 1240 }, "TRY002": { "limit": 528 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 8064e63f1aa..ab8198304bb 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 23332 + "limit": 23256 }, "LIT002": { "limit": 27213 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16792 + "limit": 16783 }, "LIT011": { "limit": 5602 From 5a5bb8c9d844870c25684e169960f1571d08e5ce Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:29:39 +0000 Subject: [PATCH 075/672] fix(proxy): stop /{provider}/v1/files from capturing /openai_passthrough The native files and batches routes declare /{provider}/v1/... and their routers are mounted before the passthrough router, so /openai_passthrough/v1/files and /openai_passthrough/v1/batches matched them with provider="openai_passthrough" and 500'd on the LlmProviders lookup instead of reaching openai_proxy_route. Move the dedicated /openai_passthrough prefix onto its own router mounted ahead of the batches and files routers. /openai/... and every other provider prefix keep their current behavior. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_passthrough_endpoints.py | 3 +- litellm/proxy/proxy_server.py | 2 + .../test_llm_pass_through_endpoints.py | 57 +++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 38da00a3bb9..baa74c19182 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -60,6 +60,7 @@ from .passthrough_endpoint_router import PassthroughEndpointRouter vertex_llm_base: Final = VertexBase() router: Final = APIRouter() +openai_passthrough_router: Final = APIRouter() default_vertex_config: Final = None passthrough_endpoint_router: Final = PassthroughEndpointRouter() @@ -1875,7 +1876,7 @@ async def vertex_proxy_route( ) -@router.api_route( +@openai_passthrough_router.api_route( "/openai_passthrough/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], tags=["OpenAI Pass-through", "pass-through"], diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fb9c4e67aad..e75277e7f0a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -522,6 +522,7 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import ( set_files_config, ) from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + openai_passthrough_router, passthrough_endpoint_router, vertex_ai_live_websocket_passthrough, ) @@ -16433,6 +16434,7 @@ app.include_router(search_router) app.include_router(image_router) app.include_router(fine_tuning_router) app.include_router(credential_router) +app.include_router(openai_passthrough_router) app.include_router(batches_router) app.include_router(openai_files_router) app.include_router(llm_passthrough_router) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 181846fe289..27d6e4c8585 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2814,6 +2814,63 @@ class TestOpenAIPassthroughRoute: assert result == {"id": "asst_123", "object": "assistant"} +def _resolve_route_name(method: str, path: str) -> str | None: + from starlette.routing import Match + + from litellm.proxy.proxy_server import app + + scope = { + "type": "http", + "method": method, + "path": path, + "headers": [], + "query_string": b"", + "root_path": "", + } + for route in app.router.routes: + if route.matches(scope)[0] == Match.FULL: + return getattr(route, "name", None) + return None + + +@pytest.mark.parametrize( + "method, path", + [ + ("POST", "/openai_passthrough/v1/files"), + ("GET", "/openai_passthrough/v1/files"), + ("GET", "/openai_passthrough/v1/files/file-abc123"), + ("DELETE", "/openai_passthrough/v1/files/file-abc123"), + ("GET", "/openai_passthrough/v1/files/file-abc123/content"), + ("POST", "/openai_passthrough/v1/batches"), + ("GET", "/openai_passthrough/v1/batches"), + ("GET", "/openai_passthrough/v1/batches/batch_abc123"), + ("POST", "/openai_passthrough/v1/batches/batch_abc123/cancel"), + ("POST", "/openai_passthrough/v1/responses"), + ], +) +def test_openai_passthrough_prefix_wins_over_native_provider_routes(method, path): + """ + /openai_passthrough exists to guarantee passthrough, so the native + /{provider}/v1/files and /{provider}/v1/batches routes must never capture it + with provider="openai_passthrough" (which 500s on the LlmProviders lookup). + """ + assert _resolve_route_name(method, path) == "openai_proxy_route" + + +@pytest.mark.parametrize( + "method, path, expected_name", + [ + ("POST", "/openai/v1/files", "create_file"), + ("GET", "/azure/v1/files", "list_files"), + ("POST", "/v1/files", "create_file"), + ("POST", "/v1/batches", "create_batch"), + ("POST", "/openai/v1/chat/completions", "openai_proxy_route"), + ], +) +def test_native_provider_routes_are_unchanged(method, path, expected_name): + assert _resolve_route_name(method, path) == expected_name + + class TestCursorProxyRoute: """Tests for the Cursor Cloud Agents pass-through route.""" From 357f90fa39d18c9a158a978ebd1ed0fecac6044d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:33:40 +0000 Subject: [PATCH 076/672] fix(proxy): scope file list pagination cursors to the caller GET /v1/files filters data down to the caller's own managed files but left first_id and last_id as the upstream page's, so a non-owner got back file ids belonging to other users even with an empty data array --- .../proxy/hooks/managed_files.py | 15 +++ .../proxy/hooks/test_managed_files.py | 100 ++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 0036603bcd1..851e202e2fb 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -1270,10 +1270,25 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) ## Filter the response to only include the files created by the user response.data = user_created_file_ids # type: ignore + self._scope_list_page_cursors(response, user_created_file_ids) return response return response return response + @staticmethod + def _scope_list_page_cursors(response: AsyncCursorPage, data: List[OpenAIFileObject]) -> None: + """Rebuild ``first_id`` / ``last_id`` from the caller-scoped page. + + The upstream cursors point at rows that were just filtered out, so + leaving them in place discloses other callers' file ids. + """ + if hasattr(response, "first_id"): + response.first_id = data[0].id if data else None + if hasattr(response, "last_id"): + response.last_id = data[-1].id if data else None + if not data and hasattr(response, "has_more"): + response.has_more = False + async def afile_retrieve( self, file_id: str, litellm_parent_otel_span: Optional[Span], llm_router=None ) -> OpenAIFileObject: diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 50af6465d06..3384c553740 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -2861,3 +2861,103 @@ async def test_same_user_different_keys_can_access_batch(): assert "batch_id" in result2 # Both keys should get the same result assert result1["batch_id"] == result2["batch_id"] + + +@pytest.mark.asyncio +async def test_file_list_cursors_are_scoped_to_the_caller(): + """A non-owner must not learn other callers' file ids through the page cursors.""" + from openai.pagination import AsyncCursorPage + from openai.types import FileObject + + from litellm.proxy._types import UserAPIKeyAuth + + owner_file = FileObject( + id="file-owner-1", + bytes=100, + created_at=1, + filename="owner.jsonl", + object="file", + purpose="batch", + status="processed", + ) + upstream_page = AsyncCursorPage[FileObject].construct( + data=[owner_file], + has_more=True, + first_id=owner_file.id, + last_id=owner_file.id, + object="list", + ) + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_many.return_value = [] + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + response = await proxy_managed_files.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth( + user_id="other-user", team_id="other-team", parent_otel_span=MagicMock() + ), + response=upstream_page, + ) + + assert response.data == [] + assert response.first_id is None + assert response.last_id is None + assert response.has_more is False + + +@pytest.mark.asyncio +async def test_file_list_cursors_follow_the_owner_scoped_page(): + from openai.pagination import AsyncCursorPage + from openai.types import FileObject + + from litellm.proxy._types import UserAPIKeyAuth + + def _raw_file(file_id: str) -> FileObject: + return FileObject( + id=file_id, + bytes=100, + created_at=1, + filename=f"{file_id}.jsonl", + object="file", + purpose="batch", + status="processed", + ) + + upstream_page = AsyncCursorPage[FileObject].construct( + data=[_raw_file("file-someone-else"), _raw_file("file-mine")], + has_more=False, + first_id="file-someone-else", + last_id="file-mine", + object="list", + ) + + managed_row = MagicMock() + managed_row.file_object = { + "id": "litellm_proxy:mine", + "bytes": 100, + "created_at": 1, + "filename": "mine.jsonl", + "object": "file", + "purpose": "batch", + "status": "processed", + } + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_many.return_value = [managed_row] + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + response = await proxy_managed_files.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth( + user_id="mine-user", parent_otel_span=MagicMock() + ), + response=upstream_page, + ) + + assert [file_object.id for file_object in response.data] == ["litellm_proxy:mine"] + assert response.first_id == "litellm_proxy:mine" + assert response.last_id == "litellm_proxy:mine" From 4a601c49a60d34d12810bd0372b062dca57d34b7 Mon Sep 17 00:00:00 2001 From: Miles Adkins Date: Thu, 6 Aug 2026 10:46:48 -0500 Subject: [PATCH 077/672] feat(fireworks_ai): full chat_template_kwargs parity with the gateway Map the remaining gateway-documented effort keys: thinking as an alias for enable_thinking (enable_thinking wins when both are present), reasoning_budget to an integer reasoning_effort (skipped when thinking is explicitly off), and low_effort=true to reasoning_effort=low (budget wins when both are set). guided_json and guided_choice response_format wrappers now include the name field (response and choice) to match the gateway wire shape. --- .../llms/fireworks_ai/chat/transformation.py | 47 ++++++++---- .../test_fireworks_ai_chat_transformation.py | 72 ++++++++++++++++++- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 3bacb3cd28e..6b763be0bfe 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -61,8 +61,32 @@ def _extract_fireworks_hidden_params(payload: dict) -> dict: return {**top_level, **per_choice} -def _json_schema_response_format(schema: object) -> Mapping[str, object]: - return {"type": "json_schema", "json_schema": {"schema": schema}} # mutable-ok: JSON request body +def _json_schema_response_format(schema: object, name: str) -> Mapping[str, object]: + return {"type": "json_schema", "json_schema": {"name": name, "schema": schema}} # mutable-ok: JSON request body + + +_EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"}) + + +def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None: + for key in keys: + value = kwargs.get(key) + if isinstance(value, bool): + return value + return None + + +def _effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object: + enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking")) + if enable_thinking is False: + return "none" + budget: Final = kwargs.get("reasoning_budget") + if isinstance(budget, (int, float)) and not isinstance(budget, bool) and budget > 0: + return int(budget) + low_effort: Final = _bool_from_kwargs(kwargs, ("low_effort",)) + if low_effort is True: + return "low" + return None _NIM_VLLM_STRIP_PARAMS: Final = frozenset( @@ -357,29 +381,28 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): type(chat_template_kwargs).__name__, ) return () - other_keys: Final = tuple(sorted(k for k in chat_template_kwargs if k != "enable_thinking")) + other_keys: Final = tuple(sorted(k for k in chat_template_kwargs if k not in _EFFORT_KWARG_KEYS)) if other_keys: verbose_logger.debug( "fireworks_ai does not support chat_template_kwargs keys %s for model=%s; dropping them.", other_keys, model, ) - if "enable_thinking" not in chat_template_kwargs: - return () if "reasoning_effort" in optional_params or "thinking" in optional_params: verbose_logger.debug( - "fireworks_ai ignoring chat_template_kwargs.enable_thinking; explicit reasoning_effort/thinking takes precedence." + "fireworks_ai ignoring chat_template_kwargs; explicit reasoning_effort/thinking takes precedence." ) return () + effort: Final = _effort_from_chat_template_kwargs(chat_template_kwargs) + if effort is None: + return () if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs.enable_thinking.", + "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", model, ) return () - if chat_template_kwargs["enable_thinking"]: - return () - return (("reasoning_effort", "none"),) + return (("reasoning_effort", effort),) @staticmethod def _translate_guided_params( @@ -396,7 +419,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): ) return () if extra_body.get("guided_json") is not None: - return (("response_format", _json_schema_response_format(extra_body["guided_json"])),) + return (("response_format", _json_schema_response_format(extra_body["guided_json"], "response")),) if extra_body.get("guided_grammar") is not None: grammar_response_format: Final = { # mutable-ok: JSON request body "type": "grammar", @@ -407,7 +430,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): "type": "string", "enum": extra_body["guided_choice"], } - return (("response_format", _json_schema_response_format(choice_schema)),) + return (("response_format", _json_schema_response_format(choice_schema, "choice")),) def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]: for tool in tools: diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 48d868b5846..e1b5d457205 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1338,6 +1338,66 @@ def test_map_extra_body_params_chat_template_kwargs_enable_thinking(): assert enabled == {} +def test_map_extra_body_params_chat_template_kwargs_thinking_alias(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"thinking": False}}}, + _REASONING_MODEL, + ) + assert result == {"reasoning_effort": "none"} + + +def test_map_extra_body_params_chat_template_kwargs_enable_thinking_wins_over_thinking(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": True, "thinking": False}}}, + _REASONING_MODEL, + ) + assert result == {} + + +def test_map_extra_body_params_chat_template_kwargs_reasoning_budget(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _REASONING_MODEL, + ) + assert result == {"reasoning_effort": 512} + + +def test_map_extra_body_params_chat_template_kwargs_budget_ignored_when_thinking_off(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False, "reasoning_budget": 512}}}, + _REASONING_MODEL, + ) + assert result == {"reasoning_effort": "none"} + + +def test_map_extra_body_params_chat_template_kwargs_low_effort(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"low_effort": True}}}, + _REASONING_MODEL, + ) + assert result == {"reasoning_effort": "low"} + + budget_wins = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"low_effort": True, "reasoning_budget": 256}}}, + _REASONING_MODEL, + ) + assert budget_wins == {"reasoning_effort": 256} + + +def test_map_extra_body_params_chat_template_kwargs_effort_keys_dropped_for_non_reasoning_model(): + config = FireworksAIConfig() + result = config.map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512, "low_effort": True}}}, + _NON_REASONING_MODEL, + ) + assert result == {} + + def test_map_extra_body_params_chat_template_kwargs_native_reasoning_effort_wins(): config = FireworksAIConfig() result = config.map_extra_body_params( @@ -1388,7 +1448,10 @@ def test_map_extra_body_params_guided_json(): {"extra_body": {"guided_json": schema}}, _REASONING_MODEL ) assert result == { - "response_format": {"type": "json_schema", "json_schema": {"schema": schema}} + "response_format": { + "type": "json_schema", + "json_schema": {"name": "response", "schema": schema}, + } } @@ -1407,7 +1470,10 @@ def test_map_extra_body_params_guided_grammar_and_choice(): assert choice == { "response_format": { "type": "json_schema", - "json_schema": {"schema": {"type": "string", "enum": ["yes", "no"]}}, + "json_schema": { + "name": "choice", + "schema": {"type": "string", "enum": ["yes", "no"]}, + }, } } @@ -1440,7 +1506,7 @@ def test_map_extra_body_params_multiple_guided_params_priority_order(): assert result == { "response_format": { "type": "json_schema", - "json_schema": {"schema": {"type": "object"}}, + "json_schema": {"name": "response", "schema": {"type": "object"}}, } } From 2d2994c9e9f45132c4ebb401a7fbf4a63d35a45f Mon Sep 17 00:00:00 2001 From: Praveena Mundolimoole <103165192+Praveena-617@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:02:44 +0530 Subject: [PATCH 078/672] fix(proxy): yaml store_prompts_in_spend_logs should take precedence over DB cached value (#35769) When store_model_in_db is true, general_settings are persisted to the LiteLLM_Config DB table. On subsequent startups and periodic reloads, _add_general_settings_from_db_config() unconditionally overwrites the in-memory general_settings with DB-cached values, including store_prompts_in_spend_logs. This means a YAML config change (e.g. store_prompts_in_spend_logs: false) deployed via CI/CD has no effect because the stale DB value (true) always wins. The admin must manually update via /config/update API after every deploy, defeating config-as-code. Fix: track which general_settings keys were explicitly set in YAML at startup (_yaml_general_settings_keys). During DB config merge, prefer the YAML value for tracked keys. The DB value is only used as fallback when YAML does not set the key, preserving the admin UI's ability to change settings at runtime. Steps to reproduce: 1. Start proxy with store_model_in_db: true, store_prompts_in_spend_logs: true 2. Change YAML to store_prompts_in_spend_logs: false, restart 3. Send a request, query LiteLLM_SpendLogs - prompts still stored 4. Check LiteLLM_Config table - DB still has true, overriding YAML Slack thread: https://dataset-jsonhackathon.slack.com/archives/C0ACUS7LM29/p1785835131860139 --- litellm/proxy/proxy_server.py | 19 ++- .../test_proxy_config_unit_test.py | 122 ++++++++++++------ 2 files changed, 99 insertions(+), 42 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 539b68c1aee..07eaed9fe45 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3909,6 +3909,10 @@ class ProxyConfig: # whether an existing request predates the prices it just fetched, and re-serving one # costs a single fetch where skipping one leaves it priced wrong indefinitely self.model_cost_map_applied_revision: int = 0 + # Keys explicitly set in the YAML config file. Used to give YAML + # precedence over stale DB-cached values for these specific keys + # during periodic config reloads (_update_general_settings). + self._yaml_general_settings_keys: set[str] = set() # mutable-ok: populated once at startup, read-only thereafter # fmt: skip def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): @@ -4839,6 +4843,11 @@ class ProxyConfig: _hc_staleness = None _hc_ignore_transient = False if general_settings: + # Record which keys were explicitly set in the YAML config file. + # These keys take precedence over DB-cached values during periodic + # reloads (see _update_general_settings). + self._yaml_general_settings_keys = set(general_settings.keys()) # mutable-ok: snapshot of YAML keys at load time # fmt: skip + ### LOAD KEY MANAGEMENT SETTINGS FIRST (needed for custom secret manager) ### key_management_settings: Final = general_settings.get("key_management_settings", None) if key_management_settings is not None: @@ -6049,7 +6058,15 @@ class ProxyConfig: ## STORE PROMPTS IN SPEND LOGS ## if "store_prompts_in_spend_logs" in _general_settings: - value = _general_settings["store_prompts_in_spend_logs"] + # If the YAML config explicitly set this key, prefer the YAML value + # over the DB-cached value. This ensures config changes deployed via + # CI/CD take effect without requiring a manual /config/update call. + # When YAML does not set this key, the DB value is used (preserving + # admin UI runtime changes). + if "store_prompts_in_spend_logs" in self._yaml_general_settings_keys: + value = general_settings.get("store_prompts_in_spend_logs") + else: + value = _general_settings["store_prompts_in_spend_logs"] # Normalize case: handle True/true/TRUE, False/false/FALSE, None/null if value is None: general_settings["store_prompts_in_spend_logs"] = None diff --git a/tests/proxy_unit_tests/test_proxy_config_unit_test.py b/tests/proxy_unit_tests/test_proxy_config_unit_test.py index 36cd08fa5f5..e6b38f31b48 100644 --- a/tests/proxy_unit_tests/test_proxy_config_unit_test.py +++ b/tests/proxy_unit_tests/test_proxy_config_unit_test.py @@ -15,9 +15,7 @@ import os # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path import asyncio import logging @@ -88,25 +86,14 @@ async def test_read_config_file_with_os_environ_vars(): # Read config proxy_config_instance = ProxyConfig() current_path = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join( - current_path, "example_config_yaml", "config_with_env_vars.yaml" - ) + config_path = os.path.join(current_path, "example_config_yaml", "config_with_env_vars.yaml") config = await proxy_config_instance.get_config(config_file_path=config_path) print(config) # Add assertions - assert ( - config["litellm_settings"]["default_internal_user_params"]["user_role"] - == "admin" - ) - assert ( - config["litellm_settings"]["s3_callback_params"]["s3_aws_access_key_id"] - == "1234567890" - ) - assert ( - config["litellm_settings"]["s3_callback_params"]["s3_aws_secret_access_key"] - == "1234567890" - ) + assert config["litellm_settings"]["default_internal_user_params"]["user_role"] == "admin" + assert config["litellm_settings"]["s3_callback_params"]["s3_aws_access_key_id"] == "1234567890" + assert config["litellm_settings"]["s3_callback_params"]["s3_aws_secret_access_key"] == "1234567890" for model in config["model_list"]: if "azure" in model["litellm_params"]["model"]: @@ -129,17 +116,13 @@ async def test_basic_include_directive(): """ proxy_config_instance = ProxyConfig() current_path = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join( - current_path, "example_config_yaml", "config_with_include.yaml" - ) + config_path = os.path.join(current_path, "example_config_yaml", "config_with_include.yaml") config = await proxy_config_instance.get_config(config_file_path=config_path) # Verify the included model list was merged assert len(config["model_list"]) > 0 - assert any( - model["model_name"] == "included-model" for model in config["model_list"] - ) + assert any(model["model_name"] == "included-model" for model in config["model_list"]) # Verify original config settings remain assert config["litellm_settings"]["callbacks"] == ["prometheus"] @@ -152,9 +135,7 @@ async def test_missing_include_file(): """ proxy_config_instance = ProxyConfig() current_path = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join( - current_path, "example_config_yaml", "config_with_missing_include.yaml" - ) + config_path = os.path.join(current_path, "example_config_yaml", "config_with_missing_include.yaml") with pytest.raises(FileNotFoundError): await proxy_config_instance.get_config(config_file_path=config_path) @@ -167,20 +148,14 @@ async def test_multiple_includes(): """ proxy_config_instance = ProxyConfig() current_path = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join( - current_path, "example_config_yaml", "config_with_multiple_includes.yaml" - ) + config_path = os.path.join(current_path, "example_config_yaml", "config_with_multiple_includes.yaml") config = await proxy_config_instance.get_config(config_file_path=config_path) # Verify models from both included files are present assert len(config["model_list"]) == 2 - assert any( - model["model_name"] == "included-model-1" for model in config["model_list"] - ) - assert any( - model["model_name"] == "included-model-2" for model in config["model_list"] - ) + assert any(model["model_name"] == "included-model-1" for model in config["model_list"]) + assert any(model["model_name"] == "included-model-2" for model in config["model_list"]) # Verify original config settings remain assert config["litellm_settings"]["callbacks"] == ["prometheus"] @@ -211,8 +186,7 @@ def test_add_callbacks_from_db_config(): # 1 instance of LangfusePromptManagement should exist in litellm.success_callback num_langfuse_instances = sum( - isinstance(callback, LangfusePromptManagement) - for callback in litellm.success_callback + isinstance(callback, LangfusePromptManagement) for callback in litellm.success_callback ) assert num_langfuse_instances == 1 assert len(litellm.success_callback) == 2 @@ -290,9 +264,7 @@ async def test_json_logs_calls_turn_on_json(): "litellm_settings": {"json_logs": True}, } - with tempfile.NamedTemporaryFile( - mode="w", suffix=".yaml", delete=False - ) as temp_file: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as temp_file: yaml.dump(config_content, temp_file) temp_file_path = temp_file.name @@ -316,3 +288,71 @@ async def test_json_logs_calls_turn_on_json(): # Cleanup os.unlink(temp_file_path) litellm.json_logs = False + + +class TestYamlStorePromptsDbOverride: + """ + Test that YAML store_prompts_in_spend_logs takes precedence over DB-cached value. + + When store_model_in_db=true, LiteLLM persists general_settings to the DB. + On periodic reloads, _update_general_settings() must NOT override + YAML-explicit values with stale DB values. + """ + + def _make_proxy_config_with_yaml_keys(self, yaml_keys: set) -> "ProxyConfig": + """Helper: create ProxyConfig with pre-populated _yaml_general_settings_keys.""" + proxy_config = ProxyConfig() + proxy_config._yaml_general_settings_keys = yaml_keys + return proxy_config + + @pytest.mark.asyncio + async def test_yaml_value_takes_precedence_over_db(self): + """When YAML sets store_prompts_in_spend_logs=false, DB value (true) should be ignored.""" + proxy_config = self._make_proxy_config_with_yaml_keys({"store_prompts_in_spend_logs"}) + + test_general_settings = {"store_prompts_in_spend_logs": False} + + with mock.patch("litellm.proxy.proxy_server.general_settings", test_general_settings): + await proxy_config._update_general_settings( + db_general_settings={"store_prompts_in_spend_logs": True}, + ) + + assert test_general_settings["store_prompts_in_spend_logs"] is False + + @pytest.mark.asyncio + async def test_db_value_used_when_yaml_does_not_set_key(self): + """When YAML does NOT set store_prompts_in_spend_logs, DB value should be used.""" + proxy_config = self._make_proxy_config_with_yaml_keys({"master_key", "database_url"}) + + test_general_settings = {"master_key": "sk-test"} + + with mock.patch("litellm.proxy.proxy_server.general_settings", test_general_settings): + await proxy_config._update_general_settings( + db_general_settings={"store_prompts_in_spend_logs": True}, + ) + + assert test_general_settings["store_prompts_in_spend_logs"] is True + + @pytest.mark.asyncio + async def test_admin_ui_change_works_when_yaml_omits_key(self): + """Admin UI change (DB update) should work when YAML doesn't set the key.""" + proxy_config = self._make_proxy_config_with_yaml_keys({"master_key"}) + + test_general_settings = {"master_key": "sk-test"} + + with mock.patch("litellm.proxy.proxy_server.general_settings", test_general_settings): + await proxy_config._update_general_settings( + db_general_settings={"store_prompts_in_spend_logs": True}, + ) + assert test_general_settings["store_prompts_in_spend_logs"] is True + + await proxy_config._update_general_settings( + db_general_settings={"store_prompts_in_spend_logs": False}, + ) + + assert test_general_settings["store_prompts_in_spend_logs"] is False + + def test_yaml_general_settings_keys_populated_on_load(self): + """_yaml_general_settings_keys should be empty on init.""" + proxy_config = ProxyConfig() + assert proxy_config._yaml_general_settings_keys == set() From b5823d5894d28130b1a8748c9edea898d4055452 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 6 Aug 2026 09:49:13 -0700 Subject: [PATCH 079/672] feat(terraform): sync provider 0.3.0 from mirror and cut 0.4.0 The provider's release gate in project-releaser publishes only when the topmost released heading in terraform/provider/CHANGELOG.md moves past the tag the mirror already carries. That heading has been 0.2.2 since 2026-05-13, so every stable release since has correctly decided there was nothing to publish and the registry has gone stale. Two things were blocking a release: 1. The mirror shipped 0.3.0 out-of-band on 2026-07-13 (pricing_base_model, BerriAI/terraform-provider-litellm#47) after the source move, so that code exists only in the mirror. The publish rsyncs monorepo -> mirror with --delete, so publishing without this port would have deleted a released feature from the registry. 2. Nothing here declared a new version. Port #47 verbatim (resource_model.go and resource_model_crud.go are now byte-identical to the mirror's released files), backfill the 0.3.0 changelog entry it shipped under, and cut 0.4.0 covering the changes made here since the source move. 0.3.0 is not reusable as the next version -- the mirror holds that tag and the publish workflow's tag guard rejects it. --- terraform/provider/CHANGELOG.md | 13 ++++++++++++ terraform/provider/docs/resources/model.md | 2 ++ terraform/provider/litellm/resource_model.go | 8 +++++++ .../provider/litellm/resource_model_crud.go | 21 +++++++++++++++++-- 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 101519c0b08..7c744f04064 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -7,13 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0] - 2026-08-06 + ### Fixed - **organization**: Send `PATCH` instead of `POST` to `/organization/update` and `/organization/member_update`, matching the methods the LiteLLM proxy serves; organization and organization member updates previously failed with a 405 +- **team_member**: Include `role` in the update payload so a role change on an existing `litellm_team_member` is applied instead of being silently dropped ### Changed - The provider source of truth moved to `terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm); this repository is now a release mirror. CI in the monorepo statically audits every endpoint the provider calls against the proxy's OpenAPI schema on every change +- **mcp_server**, **vector_store**: `env` and `litellm_params` are now marked sensitive, so they are redacted from plan/apply output, and they are no longer read back from the API into state — the configured value is authoritative. If the proxy returns values that differ from the configuration, that drift is no longer surfaced on refresh +- Dependency updates: `grpc` and `golang.org/x` modules + +## [0.3.0] - 2026-07-13 + +Released from the mirror repository before the source move was complete; this entry backfills it in the monorepo changelog. + +### Added + +- **model**: Add optional `pricing_base_model` attribute that sets `model_info.base_model` (the cost-map lookup key) independently of routing. Deployments whose routing name differs from the pricing key (for example Azure Data Zone, routed as `azure/gpt-4.1` but priced via `us/gpt-4.1-2025-04-14`) can now be billed correctly without breaking routing. When unset, behavior is unchanged and `base_model` continues to drive both routing and pricing (#47) ## [0.2.2] - 2026-05-13 diff --git a/terraform/provider/docs/resources/model.md b/terraform/provider/docs/resources/model.md index 5a46fe2f073..0409b48b391 100644 --- a/terraform/provider/docs/resources/model.md +++ b/terraform/provider/docs/resources/model.md @@ -118,6 +118,8 @@ The following arguments are supported: * `base_model` - (Required) string. The actual model identifier from the provider (e.g., "gpt-4", "claude-2"). +* `pricing_base_model` - (Optional) string. A pricing key fed to `model_info.base_model` **independently of routing**. When set, `litellm_params.model` still routes via `base_model`, but LiteLLM looks up cost against this key. Useful when the routing/deployment name differs from the cost-map key — e.g. an Azure deployment routed as `azure/gpt-4.1` whose real tier is Data Zone: set `pricing_base_model = "us/gpt-4.1-2025-04-14"` so it is billed at the Data Zone rate. When unset, `base_model` drives pricing as before. + * `litellm_credential_name` - (Optional) string. Name of a LiteLLM credential to use for this model. * `tier` - (Optional) string. The usage tier for this model. Valid values are `"free"` or `"paid"`. Default: `"free"`. diff --git a/terraform/provider/litellm/resource_model.go b/terraform/provider/litellm/resource_model.go index 2858b6e763d..4bad057871d 100644 --- a/terraform/provider/litellm/resource_model.go +++ b/terraform/provider/litellm/resource_model.go @@ -73,6 +73,14 @@ func resourceLiteLLMModel() *schema.Resource { Type: schema.TypeString, Required: true, }, + "pricing_base_model": { + // Optional pricing key fed to model_info.base_model, DECOUPLED + // from routing. When set, litellm_params.model still routes via + // base_model, but cost is looked up against this key (e.g. + // "us/gpt-4.1-2025-04-14" for Azure Data Zone pricing). + Type: schema.TypeString, + Optional: true, + }, "tier": { Type: schema.TypeString, Optional: true, diff --git a/terraform/provider/litellm/resource_model_crud.go b/terraform/provider/litellm/resource_model_crud.go index 40766c8e312..fc5d5b09dd5 100644 --- a/terraform/provider/litellm/resource_model_crud.go +++ b/terraform/provider/litellm/resource_model_crud.go @@ -68,6 +68,14 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e baseModel := d.Get("base_model").(string) modelName := fmt.Sprintf("%s/%s", customLLMProvider, baseModel) + // Pricing base_model, decoupled from routing. When pricing_base_model is + // set it feeds model_info.base_model (the cost-lookup key) WITHOUT changing + // the routing string above; otherwise base_model drives pricing as before. + pricingBaseModel := baseModel + if v, ok := d.GetOk("pricing_base_model"); ok && v.(string) != "" { + pricingBaseModel = v.(string) + } + // Generate a UUID for new models modelID := d.Id() if !isUpdate { @@ -240,7 +248,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e ModelInfo: ModelInfo{ ID: modelID, DBModel: true, - BaseModel: baseModel, + BaseModel: pricingBaseModel, Tier: d.Get("tier").(string), Mode: d.Get("mode").(string), TeamID: d.Get("team_id").(string), @@ -306,7 +314,16 @@ func resourceLiteLLMModelRead(d *schema.ResourceData, m interface{}) error { d.Set("rpm", GetIntValue(modelResp.LiteLLMParams.RPM, d.Get("rpm").(int))) d.Set("model_api_base", GetStringValue(modelResp.LiteLLMParams.APIBase, d.Get("model_api_base").(string))) d.Set("api_version", GetStringValue(modelResp.LiteLLMParams.APIVersion, d.Get("api_version").(string))) - d.Set("base_model", GetStringValue(modelResp.ModelInfo.BaseModel, d.Get("base_model").(string))) + // base_model / pricing_base_model read-back. When pricing_base_model is + // configured, model_info.base_model holds the PRICING key, so recover the + // routing base_model from state (not returned by the API) and read + // pricing_base_model from model_info. + if pbm, ok := d.GetOk("pricing_base_model"); ok && pbm.(string) != "" { + d.Set("base_model", d.Get("base_model").(string)) + d.Set("pricing_base_model", GetStringValue(modelResp.ModelInfo.BaseModel, pbm.(string))) + } else { + d.Set("base_model", GetStringValue(modelResp.ModelInfo.BaseModel, d.Get("base_model").(string))) + } d.Set("tier", GetStringValue(modelResp.ModelInfo.Tier, d.Get("tier").(string))) d.Set("mode", GetStringValue(modelResp.ModelInfo.Mode, d.Get("mode").(string))) d.Set("team_id", GetStringValue(modelResp.ModelInfo.TeamID, d.Get("team_id").(string))) From 845680ed1dc1e2f4b6c4493a00289e2f9422bbf0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:50:09 -0700 Subject: [PATCH 080/672] test(proxy): unit test batch file id mapping helpers directly --- .../test_common_utils.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/test_litellm/proxy/openai_files_endpoint/test_common_utils.py diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_common_utils.py b/tests/test_litellm/proxy/openai_files_endpoint/test_common_utils.py new file mode 100644 index 00000000000..4a021627c3e --- /dev/null +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_common_utils.py @@ -0,0 +1,97 @@ +import os +import sys +from types import MappingProxyType +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.proxy.openai_files_endpoints.common_utils import ( + apply_unified_file_ids, + map_raw_file_ids_to_unified, +) +from litellm.types.utils import LiteLLMBatch + + +def _batch(input_file_id, output_file_id, error_file_id) -> LiteLLMBatch: + return LiteLLMBatch( + id="batch-1", + completion_window="24h", + created_at=1234567890, + endpoint="/v1/chat/completions", + input_file_id=input_file_id, + object="batch", + status="cancelled", + output_file_id=output_file_id, + error_file_id=error_file_id, + ) + + +@pytest.mark.asyncio +async def test_map_raw_file_ids_to_unified_empty_ids_skips_db(): + prisma_client = MagicMock() + + assert await map_raw_file_ids_to_unified(frozenset(), prisma_client) == {} + + prisma_client.db.litellm_managedfiletable.find_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_map_raw_file_ids_to_unified_no_prisma_client_returns_empty(): + assert await map_raw_file_ids_to_unified(frozenset({"file-raw-1"}), None) == {} + + +@pytest.mark.asyncio +async def test_map_raw_file_ids_to_unified_bulk_queries_and_filters_to_requested_ids(): + row_a = MagicMock( + unified_file_id="unified-a", + flat_model_file_ids=["file-raw-a", "file-raw-other"], + ) + row_b = MagicMock(unified_file_id="unified-b", flat_model_file_ids=["file-raw-b"]) + prisma_client = MagicMock() + prisma_client.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[row_a, row_b]) + + mapping = await map_raw_file_ids_to_unified( + frozenset({"file-raw-b", "file-raw-a", "file-raw-missing"}), prisma_client + ) + + prisma_client.db.litellm_managedfiletable.find_many.assert_awaited_once_with( + where={"flat_model_file_ids": {"hasSome": ["file-raw-a", "file-raw-b", "file-raw-missing"]}} + ) + assert dict(mapping) == {"file-raw-a": "unified-a", "file-raw-b": "unified-b"} + + +def test_apply_unified_file_ids_swaps_only_mapped_ids(): + batch = _batch(input_file_id="file-raw-in", output_file_id="file-raw-out", error_file_id=None) + + apply_unified_file_ids(batch, MappingProxyType({"file-raw-out": "unified-out"})) + + assert batch.input_file_id == "file-raw-in" + assert batch.output_file_id == "unified-out" + assert batch.error_file_id is None + + +def test_apply_unified_file_ids_swaps_all_three_ids(): + batch = _batch( + input_file_id="file-raw-in", + output_file_id="file-raw-out", + error_file_id="file-raw-err", + ) + + apply_unified_file_ids( + batch, + MappingProxyType( + { + "file-raw-in": "unified-in", + "file-raw-out": "unified-out", + "file-raw-err": "unified-err", + } + ), + ) + + assert (batch.input_file_id, batch.output_file_id, batch.error_file_id) == ( + "unified-in", + "unified-out", + "unified-err", + ) From 495eb7e7f428a64ebfb9b57004026dc7739dcbc1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 6 Aug 2026 11:26:32 -0700 Subject: [PATCH 081/672] test(router): assert the auto-router max_input_chars kwarg PR #35956 added the max_input_chars passthrough to the AutoRouter constructor but left this mock assertion in tests/router_unit_tests unchanged, so test_init_auto_router_deployment_success has been failing on litellm_internal_staging ever since. The passthrough itself is intentional and its behaviour is already covered by TestAutoRouterMaxInputCharsWiring in tests/test_litellm, so only the stale expected kwargs need updating. Assert the shared constant rather than the literal 2000 so tuning the default does not break this test again. --- tests/router_unit_tests/test_router_helper_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index bcc70fae67c..0655763d41b 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -15,6 +15,7 @@ from unittest.mock import patch, MagicMock, AsyncMock from create_mock_standard_logging_payload import create_standard_logging_payload from litellm.types.utils import StandardLoggingPayload from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS @pytest.fixture @@ -1816,6 +1817,7 @@ def test_init_auto_router_deployment_success(mock_auto_router, model_list): default_model="gpt-5-mini", embedding_model="text-embedding-3-small", litellm_router_instance=router, + max_input_chars=DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ) # Verify the auto-router was added to the router's auto_routers dict From 1d8a642e0683e13be122c532436e0919d6c540f5 Mon Sep 17 00:00:00 2001 From: heathriel Date: Wed, 22 Jul 2026 08:41:01 -0700 Subject: [PATCH 082/672] fix(fireworks_ai): support router slugs via routers/ prefix Bare fireworks_ai/ only resolved to accounts/fireworks/models/, so Fireworks routers (served at accounts/fireworks/routers/, e.g. glm-latest and firerouter) could not be reached without passing the full resource id. Add a shared resolve_fireworks_resource_name helper that maps an explicit routers/ or models/ segment to the right resource path, keeps the existing -fast router heuristic, and defaults bare slugs to models/ for backward compatibility. Wire it into both the chat and text-completion transforms, which had drifted (completion lacked router handling entirely) --- .../llms/fireworks_ai/chat/transformation.py | 18 ++++---- litellm/llms/fireworks_ai/common_utils.py | 11 +++++ .../fireworks_ai/completion/transformation.py | 7 +-- .../test_fireworks_ai_chat_transformation.py | 43 ++++++++++++++++++ ..._fireworks_ai_completion_transformation.py | 34 ++++++++++++++ .../test_fireworks_ai_common_utils.py | 45 +++++++++++++++++++ type-discipline-budget.json | 2 +- 7 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py create mode 100644 tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index a796aa47b70..26f0caefacd 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -39,7 +39,11 @@ from ...openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) -from ..common_utils import FireworksAIException, FireworksAIMixin +from ..common_utils import ( + FireworksAIException, + FireworksAIMixin, + resolve_fireworks_resource_name, +) def _extract_fireworks_hidden_params(payload: dict) -> dict: @@ -459,12 +463,10 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): litellm_params: dict, headers: dict, ) -> dict: - if not model.startswith("accounts/") and "#" not in model: - if model.endswith("-fast"): - model = f"accounts/fireworks/routers/{model}" - else: - model = f"accounts/fireworks/models/{model}" - messages = self._transform_messages_helper(messages=messages, model=model, litellm_params=litellm_params) + resolved_model: Final = resolve_fireworks_resource_name(model) + messages = self._transform_messages_helper( + messages=messages, model=resolved_model, litellm_params=litellm_params + ) if "tools" in optional_params and optional_params["tools"] is not None: tools: Final = self._transform_tools(tools=optional_params["tools"]) optional_params["tools"] = tools @@ -478,7 +480,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): "include_usage": True, } return super().transform_request( - model=model, + model=resolved_model, messages=messages, optional_params=optional_params, litellm_params=litellm_params, diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 143dd151027..e07e7a26f9e 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -29,6 +29,17 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None: return None +def resolve_fireworks_resource_name(model: str) -> str: + stripped: Final = model.removeprefix("fireworks_ai/") + if stripped.startswith("accounts/") or "#" in stripped: + return stripped + if stripped.startswith(("routers/", "models/")): + return f"accounts/fireworks/{stripped}" + if stripped.endswith("-fast"): + return f"accounts/fireworks/routers/{stripped}" + return f"accounts/fireworks/models/{stripped}" + + class FireworksAIMixin: """ Common Base Config functions across Fireworks AI Endpoints diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index c141e097d3a..c460510f39c 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -4,7 +4,7 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from ...base_llm.completion.transformation import BaseTextCompletionConfig from ...openai.completion.utils import _transform_prompt -from ..common_utils import FireworksAIMixin +from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig): @@ -50,11 +50,8 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig ) -> dict: prompt: Final = _transform_prompt(messages=messages) - if not model.startswith("accounts/") and "#" not in model: - model = f"accounts/fireworks/models/{model}" - data: Final = { - "model": model, + "model": resolve_fireworks_resource_name(model), "prompt": prompt, **optional_params, } diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 94945ed4bfb..87908ef60c3 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1282,3 +1282,46 @@ def test_streaming_surfaces_fireworks_response_fields(): assert surfaced["fireworks_raw_outputs"] == [raw_output] assert surfaced["fireworks_perf_metrics"] == {"prompt-tokens": 5} assert surfaced["fireworks_prompt_token_ids"] == [1, 2, 3] + + +def test_transform_request_routes_router_slug(): + config = FireworksAIConfig() + + data = config.transform_request( + model="routers/glm-latest", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert data["model"] == "accounts/fireworks/routers/glm-latest" + + +def test_transform_request_bare_slug_stays_model(): + config = FireworksAIConfig() + + data = config.transform_request( + model="glm-4p6", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert data["model"] == "accounts/fireworks/models/glm-4p6" + + +def test_transform_request_direct_route_passthrough(): + config = FireworksAIConfig() + model = "accounts/fireworks/models/qwen2p5-coder-7b#accounts/gitlab/deployments/2fb7764c" + + data = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert data["model"] == model diff --git a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py b/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py new file mode 100644 index 00000000000..996f1fd975b --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py @@ -0,0 +1,34 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.fireworks_ai.completion.transformation import ( + FireworksAITextCompletionConfig, +) + + +def test_transform_text_completion_request_routes_router_slug(): + config = FireworksAITextCompletionConfig() + + data = config.transform_text_completion_request( + model="routers/glm-latest", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + headers={}, + ) + + assert data["model"] == "accounts/fireworks/routers/glm-latest" + + +def test_transform_text_completion_request_bare_slug_stays_model(): + config = FireworksAITextCompletionConfig() + + data = config.transform_text_completion_request( + model="glm-4p6", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + headers={}, + ) + + assert data["model"] == "accounts/fireworks/models/glm-4p6" diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py new file mode 100644 index 00000000000..4af395baf41 --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py @@ -0,0 +1,45 @@ +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.llms.fireworks_ai.common_utils import resolve_fireworks_resource_name + + +@pytest.mark.parametrize( + "model, expected", + [ + ("routers/glm-latest", "accounts/fireworks/routers/glm-latest"), + ("routers/firerouter", "accounts/fireworks/routers/firerouter"), + ("fireworks_ai/routers/glm-latest", "accounts/fireworks/routers/glm-latest"), + ("models/glm-4p6", "accounts/fireworks/models/glm-4p6"), + ("fireworks_ai/models/glm-4p6", "accounts/fireworks/models/glm-4p6"), + ("glm-4p6", "accounts/fireworks/models/glm-4p6"), + ("fireworks_ai/glm-4p6", "accounts/fireworks/models/glm-4p6"), + ("kimi-k2p6-fast", "accounts/fireworks/routers/kimi-k2p6-fast"), + ( + "accounts/fireworks/routers/glm-latest", + "accounts/fireworks/routers/glm-latest", + ), + ( + "accounts/fireworks/models/glm-4p6", + "accounts/fireworks/models/glm-4p6", + ), + ( + "fireworks_ai/accounts/fireworks/routers/glm-latest", + "accounts/fireworks/routers/glm-latest", + ), + ( + "accounts/fireworks/models/qwen2p5-coder-7b#accounts/gitlab/deployments/2fb7764c", + "accounts/fireworks/models/qwen2p5-coder-7b#accounts/gitlab/deployments/2fb7764c", + ), + ( + "glm-4p6#accounts/gitlab/deployments/2fb7764c", + "glm-4p6#accounts/gitlab/deployments/2fb7764c", + ), + ], +) +def test_resolve_fireworks_resource_name(model, expected): + assert resolve_fireworks_resource_name(model) == expected diff --git a/type-discipline-budget.json b/type-discipline-budget.json index ab8198304bb..d9038e20df9 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -30,6 +30,6 @@ "limit": 16783 }, "LIT011": { - "limit": 5602 + "limit": 5599 } } From f5d98c0b8ce15164f25b880258fb88c24f03baeb Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 6 Aug 2026 14:25:11 -0700 Subject: [PATCH 083/672] feat(ui): migrate playground chat controls toward shadcn Continue the Playground Chat Ant Design/Tremor migration: shared MultiSelect, upload validation with semantic file inputs, collapsible message widgets, and AdditionalModelSettings on Base UI controls --- .../components/chat_ui/A2AMetrics.tsx | 237 +++++++------ .../chat_ui/AdditionalModelSettings.tsx | 224 +++++++----- .../components/chat_ui/ChatImageUpload.tsx | 87 +++-- .../components/chat_ui/ChatMessageBubble.tsx | 8 +- .../playground/components/chat_ui/ChatUI.tsx | 251 +++++++------- .../chat_ui/CodeInterpreterOutput.tsx | 215 ++++++------ .../chat_ui/CodeInterpreterTool.tsx | 27 +- .../components/chat_ui/EndpointSelector.tsx | 14 +- .../components/chat_ui/FilePreviewCard.tsx | 17 +- .../chat_ui/ResponsesImageUpload.tsx | 83 +++-- .../chat_ui/SearchResultsDisplay.tsx | 168 ++++----- .../components/chat_ui/SessionManagement.tsx | 73 ++-- .../chat_ui/uploadValidation.test.ts | 78 +++++ .../components/chat_ui/uploadValidation.ts | 97 ++++++ .../src/app/(dashboard)/playground/page.tsx | 10 +- .../components/chat_ui/MCPEventsDisplay.tsx | 323 ++++++++---------- .../components/chat_ui/ReasoningContent.tsx | 117 +++---- .../components/chat_ui/ResponseMetrics.tsx | 131 +++---- .../guardrails/GuardrailSelector.tsx | 13 +- .../src/components/llm_calls/fetch_models.tsx | 20 +- .../components/policies/PolicySelector.tsx | 13 +- .../src/components/shared/MultiSelect.tsx | 119 +++++++ .../src/components/shared/SearchSelect.tsx | 4 +- .../components/tag_management/TagSelector.tsx | 17 +- .../src/components/ui/combobox.tsx | 7 +- .../VectorStoreSelector.tsx | 17 +- 26 files changed, 1414 insertions(+), 956 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/uploadValidation.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/uploadValidation.ts create mode 100644 ui/litellm-dashboard/src/components/shared/MultiSelect.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/A2AMetrics.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/A2AMetrics.tsx index 004a513f061..6ddfe1442f5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/A2AMetrics.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/A2AMetrics.tsx @@ -1,17 +1,19 @@ import React, { useState } from "react"; -import { Tooltip, Button } from "antd"; import { - CheckCircleOutlined, - ClockCircleOutlined, - LoadingOutlined, - ExclamationCircleOutlined, - CopyOutlined, - DownOutlined, - RightOutlined, - LinkOutlined, - FileTextOutlined, - RobotOutlined, -} from "@ant-design/icons"; + Bot, + CheckCircle, + ChevronDown, + ChevronRight, + CircleAlert, + Clock, + Copy, + FileText, + Link, + LoaderCircle, +} from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; export interface A2ATaskMetadata { taskId?: string; @@ -21,7 +23,7 @@ export interface A2ATaskMetadata { timestamp?: string; message?: string; }; - metadata?: Record; + metadata?: Record; } interface A2AMetricsProps { @@ -33,15 +35,15 @@ interface A2AMetricsProps { const getStatusIcon = (state?: string) => { switch (state) { case "completed": - return ; + return ; case "working": case "submitted": - return ; + return ; case "failed": case "canceled": - return ; + return ; default: - return ; + return ; } }; @@ -91,7 +93,7 @@ const A2AMetrics: React.FC = ({ a2aMetadata, timeToFirstToken,
{/* A2A Metadata Header */}
- + A2A Metadata
@@ -109,28 +111,33 @@ const A2AMetrics: React.FC = ({ a2aMetadata, timeToFirstToken, {/* Timestamp */} {formattedTime && ( - - - + + }> + {formattedTime} - + + {status?.timestamp} )} {/* Latency */} {totalLatency !== undefined && ( - - - + + }> + {(totalLatency / 1000).toFixed(2)}s - + + Total latency )} {/* Time to first token */} {timeToFirstToken !== undefined && ( - - TTFT: {(timeToFirstToken / 1000).toFixed(2)}s + + }> + TTFT: {(timeToFirstToken / 1000).toFixed(2)}s + + Time to first token )}
@@ -139,95 +146,133 @@ const A2AMetrics: React.FC = ({ a2aMetadata, timeToFirstToken,
{/* Task ID */} {taskId && ( - - copyToClipboard(taskId)} + + copyToClipboard(taskId)} + aria-label={`Copy task ID ${taskId}`} + /> + } > - + Task: {truncateId(taskId)} - - + + + Click to copy: {taskId} )} {/* Context/Session ID */} {contextId && ( - - copyToClipboard(contextId)} + + copyToClipboard(contextId)} + aria-label={`Copy session ID ${contextId}`} + /> + } > - + Session: {truncateId(contextId)} - - + + + Click to copy: {contextId} )} {/* Details toggle */} {(metadata || status?.message) && ( - + + + } + > + {showDetails ? : } + Details + + )}
{/* Expandable details panel */} - {showDetails && ( -
- {/* Status message */} - {status?.message && ( -
- Status Message: - {status.message} -
- )} + + +
+ {/* Status message */} + {status?.message && ( +
+ Status Message: + {status.message} +
+ )} - {/* Full IDs */} - {taskId && ( -
- Task ID: - - {taskId} - - copyToClipboard(taskId)} - /> -
- )} + {/* Full IDs */} + {taskId && ( +
+ Task ID: + + {taskId} + + +
+ )} - {contextId && ( -
- Session ID: - - {contextId} - - copyToClipboard(contextId)} - /> -
- )} + {contextId && ( +
+ Session ID: + + {contextId} + + +
+ )} - {/* Metadata fields */} - {metadata && Object.keys(metadata).length > 0 && ( -
- Custom Metadata: - 
-                {JSON.stringify(metadata, null, 2)}
-
-
- )} -
- )} + {/* Metadata fields */} + {metadata && Object.keys(metadata).length > 0 && ( +
+ Custom Metadata: + 
+                  {JSON.stringify(metadata, null, 2)}
+
+
+ )} +
+ + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx index d4320110c4c..4deb7051954 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx @@ -1,7 +1,10 @@ -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Text } from "@tremor/react"; -import { Checkbox, InputNumber, Popover, Slider, Tooltip, Typography } from "antd"; -import React, { useEffect, useState } from "react"; +import { Info } from "lucide-react"; +import React, { useEffect, useId, useState } from "react"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/cva.config"; interface AdditionalModelSettingsProps { temperature?: number; @@ -17,6 +20,10 @@ interface AdditionalModelSettingsProps { showAdvancedParams?: boolean; } +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + const AdditionalModelSettings: React.FC = ({ temperature = 1.0, maxTokens = 2048, @@ -36,7 +43,12 @@ const AdditionalModelSettings: React.FC = ({ const [localTemperature, setLocalTemperature] = useState(temperature); const [localMaxTokens, setLocalMaxTokens] = useState(maxTokens); - // Sync local state with props when they change + const streamingId = useId(); + const advancedId = useId(); + const fallbacksId = useId(); + const temperatureId = useId(); + const maxTokensId = useId(); + useEffect(() => { setLocalTemperature(temperature); }, [temperature]); @@ -45,21 +57,18 @@ const AdditionalModelSettings: React.FC = ({ setLocalMaxTokens(maxTokens); }, [maxTokens]); - const handleTemperatureChange = (value: number | null) => { - const newValue = value ?? 1.0; + const handleTemperatureChange = (value: number) => { + const newValue = clamp(Number.isFinite(value) ? value : 1.0, 0, 2); setLocalTemperature(newValue); onTemperatureChange?.(newValue); }; - const handleMaxTokensChange = (value: number | null) => { - const newValue = value ?? 1000; + const handleMaxTokensChange = (value: number) => { + const newValue = clamp(Number.isFinite(value) ? Math.round(value) : 1000, 1, 32768); setLocalMaxTokens(newValue); onMaxTokensChange?.(newValue); }; - const disabledOpacity = useAdvancedParams ? 1 : 0.4; - const disabledTextColor = useAdvancedParams ? "text-gray-700" : "text-gray-400"; - const handleUseAdvancedParamsChange = (checked: boolean) => { if (onUseAdvancedParamsChange) { onUseAdvancedParamsChange(checked); @@ -68,129 +77,176 @@ const AdditionalModelSettings: React.FC = ({ } }; + const disabledTextColor = useAdvancedParams ? "text-gray-700" : "text-gray-400"; + return ( -
+
{onStreamingChange && ( -
- onStreamingChange(e.target.checked)}> - Stream responses - - - +
+ onStreamingChange(checked === true)} + aria-label="Stream responses" + /> + + + + + + + Streams the answer token by token. Uncheck to send a non-streaming request and render the full response at + once. +
)} {showAdvancedParams && ( - handleUseAdvancedParamsChange(e.target.checked)}> - Use Advanced Parameters - +
+ handleUseAdvancedParamsChange(checked === true)} + aria-label="Use Advanced Parameters" + /> + +
)} {onMockTestFallbacksChange && ( -
- onMockTestFallbacksChange(e.target.checked)}> - Simulate failure to test fallbacks - - - - Causes the first request to fail so the router tries fallbacks (if configured). Use this to verify - your fallback setup. - - - Behavior can differ when keys, teams, or router settings are configured.{" "} - - Learn more - - -
- } - > - +
+ onMockTestFallbacksChange(checked === true)} + aria-label="Simulate failure to test fallbacks" + /> + + + + + + +

+ Causes the first request to fail so the router tries fallbacks (if configured). Use this to verify your + fallback setup. +

+

+ Behavior can differ when keys, teams, or router settings are configured.{" "} + + Learn more + +

+
)} {showAdvancedParams && ( -
+
-
+
- Temperature - - + + + + + + + Controls randomness. Lower values make output more deterministic, higher values more creative. +
- handleTemperatureChange(Number(event.target.value))} />
- handleTemperatureChange(Number(event.target.value))} /> +
+ 0 + 1.0 + 2.0 +
-
+
- Max Tokens - - + + + + + + + Maximum number of tokens to generate in the response. +
- handleMaxTokensChange(Number(event.target.value))} />
- handleMaxTokensChange(Number(event.target.value))} /> +
+ 1 + 32768 +
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatImageUpload.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatImageUpload.tsx index 55527d997ac..6f210118281 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatImageUpload.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatImageUpload.tsx @@ -1,43 +1,70 @@ -import React from "react"; -import { Upload, Tooltip } from "antd"; -import { PaperClipOutlined } from "@ant-design/icons"; - -const { Dragger } = Upload; +import React, { useId, useRef } from "react"; +import { Paperclip } from "lucide-react"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { Button } from "@/components/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { CHAT_ATTACHMENT_ACCEPT, validateChatAttachment } from "./uploadValidation"; interface ChatImageUploadProps { chatUploadedImage: File | null; chatImagePreviewUrl: string | null; - onImageUpload: (file: File) => false; + onImageUpload: (file: File) => void; onRemoveImage: () => void; + disabled?: boolean; } -const ChatImageUpload: React.FC = ({ - chatUploadedImage, - chatImagePreviewUrl, - onImageUpload, - onRemoveImage, -}) => { +const ChatImageUpload: React.FC = ({ chatUploadedImage, onImageUpload, disabled = false }) => { + const inputRef = useRef(null); + const inputId = useId(); + + if (chatUploadedImage) { + return null; + } + + const handleFileChange = (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + event.target.value = ""; + if (!file) { + return; + } + const result = validateChatAttachment(file); + if (!result.ok) { + NotificationsManager.error(result.error); + return; + } + onImageUpload(file); + }; + return ( <> - {/* Subtle upload button - only show when no image */} - {!chatUploadedImage && ( - - - - - - )} + variant="ghost" + size="icon-sm" + disabled={disabled} + aria-label="Attach image or PDF" + className="text-gray-400 hover:text-gray-600" + onClick={() => inputRef.current?.click()} + /> + } + > + + + Attach image or PDF + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatMessageBubble.tsx index 8e71017a7b5..c438b4982bf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatMessageBubble.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatMessageBubble.tsx @@ -41,9 +41,9 @@ function ChatMessageBubble({ const isUser = message.role === "user"; return ( -
+
{/* Header: role icon + name + model badge */} -
+
{message.role} {message.role === "assistant" && message.model && ( - + {message.model} )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index 57ff7906eda..5edcbe84aa8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -66,7 +66,16 @@ import { MessageType } from "@/components/chat_ui/types"; import { useCodeInterpreter } from "../../hooks/useCodeInterpreter"; import { useChatHistory } from "../../hooks/useChatHistory"; import { getSecureItem, setSecureItem } from "@/utils/secureStorage"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Select as ShadcnSelect, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; +import { + AUDIO_ACCEPT, + IMAGE_EDIT_ACCEPT, + validateAudioFile, + validateChatAttachment, + validateImageEditFile, +} from "./uploadValidation"; const { TextArea } = Input; const { Dragger } = Upload; @@ -177,6 +186,8 @@ const ChatUI: React.FC = ({ const [selectedModel, setSelectedModel] = useState(simplified ? fixedModel : undefined); const [showCustomModelInput, setShowCustomModelInput] = useState(false); const [modelInfo, setModelInfo] = useState([]); + const [isLoadingModels, setIsLoadingModels] = useState(false); + const [modelLoadError, setModelLoadError] = useState(false); const [agentInfo, setAgentInfo] = useState([]); const [selectedAgent, setSelectedAgent] = useState(undefined); const debouncedSetSelectedModel = useDebouncedCallback((value: string) => setSelectedModel(value), { @@ -388,17 +399,17 @@ const ChatUI: React.FC = ({ ]); useEffect(() => { - let userApiKey = apiKeySource === "session" ? accessToken : apiKey; - if (!userApiKey || !token || !userRole || !userID) { + const userApiKey = apiKeySource === "session" ? accessToken : apiKey.trim(); + if (!userApiKey) { + setModelInfo([]); + setModelLoadError(false); return; } - // Fetch model info and set the default selected model (skip in simplified mode; we use fixedModel) const loadModels = async () => { + setIsLoadingModels(true); + setModelLoadError(false); try { - if (!userApiKey) { - return; - } const uniqueModels = await fetchAvailableModels(userApiKey); setModelInfo(uniqueModels); @@ -412,6 +423,10 @@ const ChatUI: React.FC = ({ } } catch (error) { console.error("Error fetching model info:", error); + setModelInfo([]); + setModelLoadError(true); + } finally { + setIsLoadingModels(false); } }; @@ -419,7 +434,7 @@ const ChatUI: React.FC = ({ loadModels(); } loadMCPServers(); - }, [accessToken, userID, userRole, apiKeySource, apiKey, token, simplified]); + }, [accessToken, apiKeySource, apiKey, simplified]); // Load tools when MCP direct mode has a server (or toolset) selected useEffect(() => { @@ -494,13 +509,35 @@ const ChatUI: React.FC = ({ } }; - const handleImageUpload = (file: File) => { - setUploadedImages((prev) => [...prev, file]); + const createBlobPreviewUrl = (file: File): string => { const rawPreviewUrl = URL.createObjectURL(file); - // Sanitize: only allow blob: URLs to prevent XSS via img src injection. - const previewUrl = rawPreviewUrl.startsWith("blob:") ? rawPreviewUrl : ""; - setImagePreviewUrls((prev) => [...prev, previewUrl]); - return false; // Prevent default upload behavior + return rawPreviewUrl.startsWith("blob:") ? rawPreviewUrl : ""; + }; + + const handleImageFiles = (files: File[]) => { + let nextCount = uploadedImages.length; + const accepted: File[] = []; + const previews: string[] = []; + for (const file of files) { + const result = validateImageEditFile(file, nextCount); + if (!result.ok) { + NotificationsManager.error(result.error); + continue; + } + accepted.push(file); + previews.push(createBlobPreviewUrl(file)); + nextCount += 1; + } + if (accepted.length === 0) { + return; + } + setUploadedImages((prev) => [...prev, ...accepted]); + setImagePreviewUrls((prev) => [...prev, ...previews]); + }; + + const handleImageUpload = (file: File): false => { + handleImageFiles([file]); + return false; }; const handleRemoveImage = (index: number) => { @@ -519,11 +556,14 @@ const ChatUI: React.FC = ({ setImagePreviewUrls([]); }; - const handleResponsesImageUpload = (file: File): false => { + const handleResponsesImageUpload = (file: File): void => { + const result = validateChatAttachment(file); + if (!result.ok) { + NotificationsManager.error(result.error); + return; + } setResponsesUploadedImage(file); - const previewUrl = URL.createObjectURL(file); - setResponsesImagePreviewUrl(previewUrl); - return false; // Prevent default upload behavior + setResponsesImagePreviewUrl(createBlobPreviewUrl(file)); }; const handleRemoveResponsesImage = () => { @@ -534,11 +574,14 @@ const ChatUI: React.FC = ({ setResponsesImagePreviewUrl(null); }; - const handleChatImageUpload = (file: File): false => { + const handleChatImageUpload = (file: File): void => { + const result = validateChatAttachment(file); + if (!result.ok) { + NotificationsManager.error(result.error); + return; + } setChatUploadedImage(file); - const previewUrl = URL.createObjectURL(file); - setChatImagePreviewUrl(previewUrl); - return false; // Prevent default upload behavior + setChatImagePreviewUrl(createBlobPreviewUrl(file)); }; const handleRemoveChatImage = () => { @@ -550,8 +593,13 @@ const ChatUI: React.FC = ({ }; const handleAudioUpload = (file: File): false => { + const result = validateAudioFile(file); + if (!result.ok) { + NotificationsManager.error(result.error); + return false; + } setUploadedAudio(file); - return false; // Prevent default upload behavior + return false; }; const handleRemoveAudio = () => { @@ -1002,8 +1050,12 @@ const ChatUI: React.FC = ({ const onModelChange = (value: string) => { setSelectedModel(value); - setShowCustomModelInput(value === "custom"); + + const model = modelInfo.find((option) => option.model_group === value); + if (model?.mode) { + setEndpointType(getEndpointType(model.mode)); + } }; // Check if the selected model is a chat model @@ -1020,35 +1072,43 @@ const ChatUI: React.FC = ({ }; const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES; + let modelEmptyText = "No models available for this key"; + if (modelLoadError) { + modelEmptyText = "Unable to load models for this key"; + } else if (apiKeySource === "custom" && !apiKey.trim()) { + modelEmptyText = "Enter a Virtual Key to load models"; + } const antIcon = ; return ( -
- -
+
+ +
{/* Left Sidebar with Controls - hidden in simplified mode */} {!simplified && ( -
+
Configurations
Virtual Key Source - { + onValueChange={(value) => { setSelectedVoice(value); sessionStorage.setItem("selectedVoice", value); }} - style={{ width: "100%" }} - className="rounded-md" - options={OPEN_AI_VOICE_SELECT_OPTIONS} - /> + > + + + + + {OPEN_AI_VOICE_SELECT_OPTIONS.map((voice) => ( + + {voice.label} + + ))} + +
)} @@ -1212,46 +1280,20 @@ const ChatUI: React.FC = ({ )} - setSelectedAgent(value)} + onValueChange={(value) => setSelectedAgent(value)} options={agentInfo.map((agent) => ({ value: agent.agent_name, label: agent.agent_name || agent.agent_id, - key: agent.agent_id, + sublabel: agent.agent_card_params?.description, }))} - style={{ width: "100%" }} - showSearch={true} - className="rounded-md" - optionLabelProp="label" - > - {agentInfo.map((agent) => ( - -
- {agent.agent_name || agent.agent_id} - {agent.agent_card_params?.description && ( - {agent.agent_card_params.description} - )} -
- - ))} - + /> {agentInfo.length === 0 && ( No agents found. Create agents via /v1/agents endpoint. @@ -1697,7 +1720,7 @@ const ChatUI: React.FC = ({ )} {/* Main Chat Area */} -
+
{endpointType === EndpointType.REALTIME ? ( = ({ /> ) : ( <> -
+
{simplified ? "Chat" : "Test Key"} -
+
= ({ )}
-
+
{chatHistory.length === 0 && (
@@ -1788,18 +1811,18 @@ const ChatUI: React.FC = ({
-
+
{/* Image Upload Section for Image Edits */} {endpointType === EndpointType.IMAGE_EDITS && (
{uploadedImages.length === 0 ? ( - +

Click or drag images to upload

- Support for PNG, JPG, JPEG formats. Multiple images supported. + Support for PNG, JPG, JPEG, GIF, WebP. Multiple images supported.

) : ( @@ -1840,12 +1863,12 @@ const ChatUI: React.FC = ({ { - const files = Array.from(e.target.files || []); - files.forEach((file) => handleImageUpload(file)); + handleImageFiles(Array.from(e.target.files || [])); + e.target.value = ""; }} />
@@ -1858,11 +1881,7 @@ const ChatUI: React.FC = ({ {endpointType === EndpointType.TRANSCRIPTION && (
{!uploadedAudio ? ( - +

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterOutput.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterOutput.tsx index c27273a116d..8dc503f369c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterOutput.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterOutput.tsx @@ -1,15 +1,10 @@ -import React, { useState, useEffect } from "react"; -import { Collapse, Spin } from "antd"; -import { - CodeOutlined, - DownloadOutlined, - FileImageOutlined, - FileTextOutlined, - LoadingOutlined, -} from "@ant-design/icons"; +import React, { useEffect, useState } from "react"; +import { Code, Download, FileImage, FileText, Loader2 } from "lucide-react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; import { getProxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; interface ContainerFileCitation { type: "container_file_citation"; @@ -27,48 +22,60 @@ interface CodeInterpreterOutputProps { accessToken: string; } -const CodeInterpreterOutput: React.FC = ({ - code, - containerId, - annotations = [], - accessToken, -}) => { +const IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif"] as const; + +function isImageFilename(filename: string | undefined): boolean { + if (!filename) { + return false; + } + const lower = filename.toLowerCase(); + return IMAGE_EXTENSIONS.some((ext) => lower.endsWith(ext)); +} + +const CodeInterpreterOutput: React.FC = ({ code, annotations = [], accessToken }) => { const [imageUrls, setImageUrls] = useState>({}); const [loadingImages, setLoadingImages] = useState>({}); + const [codeOpen, setCodeOpen] = useState(false); const proxyBaseUrl = getProxyBaseUrl(); - // Fetch images from container files API useEffect(() => { + const createdUrls: string[] = []; + let cancelled = false; + const fetchImages = async () => { for (const annotation of annotations) { - const isImage = - annotation.filename?.toLowerCase().endsWith(".png") || - annotation.filename?.toLowerCase().endsWith(".jpg") || - annotation.filename?.toLowerCase().endsWith(".jpeg") || - annotation.filename?.toLowerCase().endsWith(".gif"); + if (!isImageFilename(annotation.filename) || !annotation.container_id || !annotation.file_id) { + continue; + } - if (isImage && annotation.container_id && annotation.file_id) { + if (!cancelled) { setLoadingImages((prev) => ({ ...prev, [annotation.file_id]: true })); + } - try { - // Fetch image content from container files API - const response = await fetch( - `${proxyBaseUrl}/v1/containers/${annotation.container_id}/files/${annotation.file_id}/content`, - { - headers: { - [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`, - }, + try { + const response = await fetch( + `${proxyBaseUrl}/v1/containers/${annotation.container_id}/files/${annotation.file_id}/content`, + { + headers: { + [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`, }, - ); + }, + ); - if (response.ok) { - const blob = await response.blob(); - const url = URL.createObjectURL(blob); + if (response.ok) { + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + createdUrls.push(url); + if (!cancelled) { setImageUrls((prev) => ({ ...prev, [annotation.file_id]: url })); + } else { + URL.revokeObjectURL(url); } - } catch (error) { - console.error("Error fetching image:", error); - } finally { + } + } catch (error) { + console.error("Error fetching image:", error); + } finally { + if (!cancelled) { setLoadingImages((prev) => ({ ...prev, [annotation.file_id]: false })); } } @@ -76,12 +83,12 @@ const CodeInterpreterOutput: React.FC = ({ }; if (annotations.length > 0 && accessToken) { - fetchImages(); + void fetchImages(); } - // Cleanup URLs on unmount return () => { - Object.values(imageUrls).forEach((url) => URL.revokeObjectURL(url)); + cancelled = true; + createdUrls.forEach((url) => URL.revokeObjectURL(url)); }; }, [annotations, accessToken, proxyBaseUrl]); @@ -112,22 +119,8 @@ const CodeInterpreterOutput: React.FC = ({ } }; - // Separate images and other files - const imageAnnotations = annotations.filter( - (a) => - a.filename?.toLowerCase().endsWith(".png") || - a.filename?.toLowerCase().endsWith(".jpg") || - a.filename?.toLowerCase().endsWith(".jpeg") || - a.filename?.toLowerCase().endsWith(".gif"), - ); - - const fileAnnotations = annotations.filter( - (a) => - !a.filename?.toLowerCase().endsWith(".png") && - !a.filename?.toLowerCase().endsWith(".jpg") && - !a.filename?.toLowerCase().endsWith(".jpeg") && - !a.filename?.toLowerCase().endsWith(".gif"), - ); + const imageAnnotations = annotations.filter((a) => isImageFilename(a.filename)); + const fileAnnotations = annotations.filter((a) => !isImageFilename(a.filename)); if (!code && annotations.length === 0) { return null; @@ -135,44 +128,46 @@ const CodeInterpreterOutput: React.FC = ({ return (
- {/* Executed Code - Collapsible */} {code && ( - - Python Code Executed - - ), - children: ( - - {code} - - ), - }, - ]} - /> + + + } + > + + Python Code Executed + + +
+ + {code} + +
+ + )} - {/* Generated Images */} {imageAnnotations.map((annotation) => ( -
+
{loadingImages[annotation.file_id] ? ( -
- } /> +
+
) : imageUrls[annotation.file_id] ? ( @@ -180,42 +175,48 @@ const CodeInterpreterOutput: React.FC = ({ {annotation.filename -
- - {annotation.filename} +
+ + - + + Download +
) : ( -
+
Image not available
)}
))} - {/* Download Links for Other Files */} {fileAnnotations.length > 0 && (
{fileAnnotations.map((annotation) => ( - +
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterTool.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterTool.tsx index d2682e3a7a8..e5744ac8e38 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterTool.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterTool.tsx @@ -1,8 +1,8 @@ import React from "react"; -import { Switch, Tooltip } from "antd"; import MessageManager from "@/components/molecules/message_manager"; -import { CodeOutlined, InfoCircleOutlined, ExclamationCircleOutlined } from "@ant-design/icons"; -import { Text } from "@tremor/react"; +import { Code, Info, TriangleAlert } from "lucide-react"; +import { Switch } from "@/components/ui/switch"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; interface CodeInterpreterToolProps { accessToken: string; @@ -49,25 +49,30 @@ const CodeInterpreterTool: React.FC = ({
- - Code Interpreter - - + + Code Interpreter + + + + + + Run Python code to generate files, charts, and analyze data. Container is created automatically. +
{!isOpenAI && (
- +
Code Interpreter is currently only supported for OpenAI models. = ({ endpointType, onEndpointChange, className }) => { return (
- + { return { label: `${guardrail.guardrail_name}`, value: guardrail.guardrail_name, }; })} - optionFilterProp="label" - showSearch - style={{ width: "100%" }} />
); diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index 0de98330c2e..24f1e038f85 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -7,6 +7,13 @@ export interface ModelGroup { mode?: string; } +interface AvailableModel { + model_group?: string | null; + model_name?: string | null; + id?: string | null; + mode?: string | null; +} + /** * Fetches available models using modelHubCall and formats them for the selection dropdown. */ @@ -15,14 +22,15 @@ export const fetchAvailableModels = async (accessToken: string): Promise 0) { - const models: ModelGroup[] = fetchedModels.data.map((item: any) => ({ - model_group: item.model_group, // Display the model_group to the user - mode: item?.mode, // Save the mode for auto-selection of endpoint type - })); + const models: ModelGroup[] = fetchedModels.data + .map((item: AvailableModel) => ({ + model_group: item.model_group || item.id || item.model_name || "", + mode: item.mode || undefined, + })) + .filter((model: ModelGroup) => model.model_group !== ""); - // Sort models alphabetically by label models.sort((a, b) => a.model_group.localeCompare(b.model_group)); - return models; + return Array.from(new Map(models.map((model) => [model.model_group, model])).values()); } return []; } catch (error) { diff --git a/ui/litellm-dashboard/src/components/policies/PolicySelector.tsx b/ui/litellm-dashboard/src/components/policies/PolicySelector.tsx index 132538d439f..1e565d938d7 100644 --- a/ui/litellm-dashboard/src/components/policies/PolicySelector.tsx +++ b/ui/litellm-dashboard/src/components/policies/PolicySelector.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; -import { Select } from "antd"; import { Policy } from "./types"; import { getPoliciesList } from "../networking"; +import { MultiSelect } from "@/components/shared/MultiSelect"; /** Prefix for policy version IDs in request body; must match backend POLICY_VERSION_ID_PREFIX. */ export const POLICY_VERSION_ID_PREFIX = "policy_"; @@ -80,22 +80,17 @@ const PolicySelector: React.FC = ({ }; return ( -
- ({ label: tag.name, value: tag.name, - title: tag.description || tag.name, + description: tag.description || undefined, }))} - optionFilterProp="label" - tokenSeparators={[","]} - maxTagCount="responsive" - allowClear - style={{ width: "100%" }} /> ); }; diff --git a/ui/litellm-dashboard/src/components/ui/combobox.tsx b/ui/litellm-dashboard/src/components/ui/combobox.tsx index 2854928140e..541ad8bb25c 100644 --- a/ui/litellm-dashboard/src/components/ui/combobox.tsx +++ b/ui/litellm-dashboard/src/components/ui/combobox.tsx @@ -83,10 +83,14 @@ function ComboboxContent({ sideOffset = 6, align = "start", alignOffset = 0, + collisionAvoidance, anchor, ...props }: ComboboxPrimitive.Popup.Props & - Pick) { + Pick< + ComboboxPrimitive.Positioner.Props, + "side" | "align" | "sideOffset" | "alignOffset" | "collisionAvoidance" | "anchor" + >) { return ( diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.tsx index 2642b74e492..d80996f6a28 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; -import { Select } from "antd"; import { VectorStore } from "./types"; import { vectorStoreListCall } from "../networking"; +import { MultiSelect } from "@/components/shared/MultiSelect"; interface VectorStoreSelectorProps { onChange: (selectedVectorStores: string[]) => void; value?: string[]; @@ -43,24 +43,19 @@ const VectorStoreSelector: React.FC = ({ }, [accessToken]); return ( -
- setApiKey(event.target.value)} + value={apiKey} + /> +
)}
-
- - Custom Proxy Base URL - +
+ {proxySettings?.LITELLM_UI_API_DOC_BASE_URL && !customProxyBaseUrl && ( )} {customProxyBaseUrl && ( )}
- { - setCustomProxyBaseUrl(value); - sessionStorage.setItem("customProxyBaseUrl", value); - }} - value={customProxyBaseUrl} - icon={ApiOutlined} - /> +
+ + { + setCustomProxyBaseUrl(event.target.value); + sessionStorage.setItem("customProxyBaseUrl", event.target.value); + }} + /> +
{customProxyBaseUrl && ( - API calls will be sent to: {customProxyBaseUrl} +

API calls will be sent to: {customProxyBaseUrl}

)}
- - Endpoint Type - + { setEndpointType(value); - // Clear model/agent selection when switching endpoint type setSelectedModel(undefined); setSelectedAgent(undefined); setShowCustomModelInput(false); setSelectedMCPDirectTool(undefined); - // For MCP direct mode, require single server (clear __all__ or multiple) if (value === EndpointType.MCP) { setSelectedMCPServers((prev) => (prev.length === 1 && prev[0] !== "__all__" ? prev : [])); } @@ -1194,13 +1279,12 @@ const ChatUI: React.FC = ({ className="mb-4" /> - {/* Voice Selector for Speech Endpoint */} {endpointType === EndpointType.SPEECH && (
- - + + { @@ -1222,7 +1306,6 @@ const ChatUI: React.FC = ({
)} - {/* Session Management Component */} = ({ />
- {/* Model Selector - shown when NOT using A2A Agents or MCP direct mode */} {endpointType !== EndpointType.A2A_AGENTS && endpointType !== EndpointType.MCP && (
- +
- Select Model + {isChatModel() || supportsStreamingToggle ? ( - + + } + > + + + +
Model Settings
= ({ streamingEnabled={streamingEnabled} onStreamingChange={supportsStreamingToggle ? setStreamingEnabled : undefined} /> - } - title="Model Settings" - trigger="click" - placement="right" - > -
= ({ ]} /> {showCustomModelInput && ( - debouncedSetSelectedModel(event.target.value)} /> )}
)} - {/* Agent Selector - shown ONLY for A2A Agents endpoint */} {endpointType === EndpointType.A2A_AGENTS && (
- - Select Agent - + = ({ }))} /> {agentInfo.length === 0 && ( - +

No agents found. Create agents via /v1/agents endpoint. - +

)}
)}
- - Tags - + = ({ />
- {/* MCP Server Selection */}
- - +
+
)} - {/* BYOK credential status for selected servers */} {selectedMCPServers.length > 0 && !selectedMCPServers.includes("__all__") && selectedMCPServers.some((serverId) => { @@ -1593,28 +1571,31 @@ const ChatUI: React.FC = ({ return (
- {serverName} requires your API key +

{serverName} requires your API key

{server.has_user_credential ? (
- - Connected + + Connected
) : ( - + )}
); @@ -1624,23 +1605,21 @@ const ChatUI: React.FC = ({
- - Vector Store - - Select vector store(s) to use for this LLM API call. You can set up your vector store{" "} - - here - - . - - } - > - +
+
= ({
- - Guardrails - - Select guardrail(s) to use for this LLM API call. You can set up your guardrails{" "} - - here - - . - - } - > - +
+
= ({
- - Policies - - Select policy/policies to apply to this LLM API call. Policies define which guardrails are - applied based on conditions. You can set up your policies{" "} - - here - - . - - } - > - +
+
= ({ />
- {/* Code Interpreter Toggle - Only for Responses endpoint */} {endpointType === EndpointType.RESPONSES && (
= ({
)} - {/* Main Chat Area */}
{endpointType === EndpointType.REALTIME ? ( = ({ ) : ( <>
- {simplified ? "Chat" : "Test Key"} +

{simplified ? "Chat" : "Test Key"}

- + {!simplified && ( - setIsGetCodeModalVisible(true)} - className="bg-gray-100 hover:bg-gray-200 text-gray-700 border-gray-300" - icon={CodeOutlined} - > + )}
{chatHistory.length === 0 && ( -
- - Start a conversation, generate an image, or handle audio +
+
)} @@ -1772,29 +1739,26 @@ const ChatUI: React.FC = ({
))} - {/* Show MCP events during loading if no assistant message exists yet */} {isLoading && mcpEvents.length > 0 && (endpointType === EndpointType.RESPONSES || endpointType === EndpointType.CHAT) && chatHistory.length > 0 && chatHistory[chatHistory.length - 1].role === "user" && ( -
+
-
+
- +
Assistant
@@ -1804,27 +1768,34 @@ const ChatUI: React.FC = ({ )} {isLoading && ( -
- +
+
)}
- {/* Image Upload Section for Image Edits */} {endpointType === EndpointType.IMAGE_EDITS && (
{uploadedImages.length === 0 ? ( - -

- -

-

Click or drag images to upload

-

+

Click or drag images to upload

+

Support for PNG, JPG, JPEG, GIF, WebP. Multiple images supported.

- + { + handleImageFiles(Array.from(event.target.files || [])); + event.target.value = ""; + }} + /> + ) : (
{uploadedImages.map((file, index) => ( @@ -1841,76 +1812,83 @@ const ChatUI: React.FC = ({ } })()} alt={`Upload preview ${index + 1}`} - className="max-w-32 max-h-32 rounded-md border border-gray-200 object-cover" + className="max-h-32 max-w-32 rounded-md border border-gray-200 object-cover" /> - + +
))} - {/* Add more images button */} -
document.getElementById("additional-image-upload")?.click()} - > -
- -

Add more

-
+
+
)}
)} - {/* Audio Upload Section for Transcriptions */} {endpointType === EndpointType.TRANSCRIPTION && (
{!uploadedAudio ? ( - -

- -

-

Click or drag audio file to upload

-

+

Click or drag audio file to upload

+

Support for MP3, MP4, MPEG, MPGA, M4A, WAV, WEBM formats. Max file size: 25 MB.

- + { + const file = event.target.files?.[0]; + if (file) handleAudioUpload(file); + event.target.value = ""; + }} + /> + ) : ( -
-
- +
+
+
- + + Remove +
)}
)} - {/* Show file previews above input when files are uploaded */} {endpointType === EndpointType.RESPONSES && responsesUploadedImage && ( = ({ /> )} - {/* Code Interpreter indicator and sample prompts when enabled */} {endpointType === EndpointType.RESPONSES && codeInterpreter.enabled && (
-
+
{isLoading ? ( <> - - Running Python code... +
- {/* Sample prompts - only show when not loading */} {!isLoading && (
{[ @@ -1961,7 +1938,8 @@ const ChatUI: React.FC = ({ ].map((prompt, idx) => (
)} - {/* Suggested prompts - show when chat is empty and not loading (skip for MCP - uses structured form) */} {chatHistory.length === 0 && !isLoading && endpointType !== EndpointType.MCP && ( -
+
{(endpointType === EndpointType.A2A_AGENTS ? ["What can you help me with?", "Tell me about yourself", "What tasks can you perform?"] : ["Write me a poem", "Explain quantum computing", "Draft a polite email requesting a meeting"] @@ -1982,7 +1959,7 @@ const ChatUI: React.FC = ({ + + + + {codeInterpreter.enabled + ? "Code Interpreter enabled (click to disable)" + : "Enable Code Interpreter"} + )}
- {/* Middle: input field or MCP structured form */} {endpointType === EndpointType.MCP && selectedMCPServers.length === 1 && selectedMCPServers[0] !== "__all__" && selectedMCPDirectTool ? ( -
+
{(() => { const rawSel = selectedMCPServers[0]; - let toolPool: any[] = []; + let toolPool: { name: string }[] = []; if (rawSel.startsWith("toolset:")) { const toolsetId = rawSel.slice("toolset:".length); const toolset = mcpToolsets.find((t) => t.toolset_id === toolsetId); @@ -2060,82 +2043,51 @@ const ChatUI: React.FC = ({ } else { toolPool = serverToolsMap[rawSel] || []; } - const mcpTool = toolPool.find((t: any) => t.name === selectedMCPDirectTool); + const mcpTool = toolPool.find((t) => t.name === selectedMCPDirectTool); return mcpTool ? ( ) : ( -
+
Loading tool schema...
); })()}
) : ( -