From d9b9676d49f570bcacf0507ac07fa08caa602e8e Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:47:22 +0000 Subject: [PATCH 1/5] fix(batches): isolate CheckBatchCost failures per job Wrap each managed-batch poll iteration in its own error boundary and make Prometheus error recording best-effort so one poisoned batch cannot abort reconciliation for unrelated jobs in the same cycle. Fixes #35357 --- .../proxy/common_utils/check_batch_cost.py | 172 ++++++------ .../proxy_unit_tests/test_check_batch_cost.py | 249 ++++++++++++++++++ 2 files changed, 341 insertions(+), 80 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index f209ab54f64..ed4a43b6f6d 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -107,7 +107,12 @@ class CheckBatchCost: prom_logger: Optional["PrometheusLogger"], error_type: str ) -> None: if prom_logger is not None: - prom_logger.record_check_batch_cost_error(error_type) + try: + prom_logger.record_check_batch_cost_error(error_type) + except Exception as prom_err: + verbose_proxy_logger.warning( + f"CheckBatchCost: failed to record {error_type} metric: {prom_err}" + ) def _resolve_job_routing( self, @@ -558,94 +563,101 @@ class CheckBatchCost: else: jobs = await self._fallback_find_jobs() for job in jobs: - routing = self._resolve_job_routing(job, prom_logger) - if routing is None: - continue - model_id, batch_id = routing - - verbose_proxy_logger.info( - f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}" - ) - try: - response = await self.llm_router.aretrieve_batch( - model=model_id, - batch_id=batch_id, - litellm_metadata={ - "user_api_key_user_id": job.created_by or "default-user-id", - "batch_ignore_default_logging": True, - }, - ) - except Exception as e: + routing = self._resolve_job_routing(job, prom_logger) + if routing is None: + continue + model_id, batch_id = routing + verbose_proxy_logger.info( - f"Skipping job {job.unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}" + f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}" ) - if prom_logger: - prom_logger.record_check_batch_cost_error("provider_retrieval_error") - continue - ## RETRIEVE THE BATCH JOB OUTPUT FILE - if ( - response.status == "completed" - and response.output_file_id is not None - ): try: - tracked = await self._track_completed_batch_cost( - job=job, - response=response, - model_id=model_id, + response = await self.llm_router.aretrieve_batch( + model=model_id, batch_id=batch_id, - prom_logger=prom_logger, - ) - except Exception as tracking_err: - verbose_proxy_logger.error( - f"CheckBatchCost: failed to track cost for batch {batch_id} " - f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}" - ) - self._record_error(prom_logger, "cost_tracking_error") - continue - if tracked is None: - continue - - # Track this job for the final metrics summary - processed_models.append(tracked) - - # mark the job as complete - try: - update_data: dict = { - "status": "complete", - "file_object": response.model_dump_json(), - } - if self._has_batch_processed_column: - update_data["batch_processed"] = True - await self.prisma_client.db.litellm_managedobjecttable.update( - where={"id": job.id}, - data=update_data, - ) - except Exception as db_err: - verbose_proxy_logger.error( - f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}" - ) - - elif response.status in ("failed", "expired", "cancelled"): - try: - update_data = { - "status": response.status, - "file_object": response.model_dump_json(), - } - if self._has_batch_processed_column: - update_data["batch_processed"] = True - await self.prisma_client.db.litellm_managedobjecttable.update( - where={"id": job.id}, - data=update_data, + litellm_metadata={ + "user_api_key_user_id": job.created_by or "default-user-id", + "batch_ignore_default_logging": True, + }, ) + except Exception as e: verbose_proxy_logger.info( - f"CheckBatchCost: marked job {job.id} as {response.status} in DB" - ) - except Exception as db_err: - verbose_proxy_logger.error( - f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}" + f"Skipping job {job.unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}" ) + self._record_error(prom_logger, "provider_retrieval_error") + continue + + ## RETRIEVE THE BATCH JOB OUTPUT FILE + if ( + response.status == "completed" + and response.output_file_id is not None + ): + try: + tracked = await self._track_completed_batch_cost( + job=job, + response=response, + model_id=model_id, + batch_id=batch_id, + prom_logger=prom_logger, + ) + except Exception as tracking_err: + verbose_proxy_logger.error( + f"CheckBatchCost: failed to track cost for batch {batch_id} " + f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}" + ) + self._record_error(prom_logger, "cost_tracking_error") + continue + if tracked is None: + continue + + # Track this job for the final metrics summary + processed_models.append(tracked) + + # mark the job as complete + try: + update_data: dict = { + "status": "complete", + "file_object": response.model_dump_json(), + } + if self._has_batch_processed_column: + update_data["batch_processed"] = True + await self.prisma_client.db.litellm_managedobjecttable.update( + where={"id": job.id}, + data=update_data, + ) + except Exception as db_err: + verbose_proxy_logger.error( + f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}" + ) + + elif response.status in ("failed", "expired", "cancelled"): + try: + update_data = { + "status": response.status, + "file_object": response.model_dump_json(), + } + if self._has_batch_processed_column: + update_data["batch_processed"] = True + await self.prisma_client.db.litellm_managedobjecttable.update( + where={"id": job.id}, + data=update_data, + ) + verbose_proxy_logger.info( + f"CheckBatchCost: marked job {job.id} as {response.status} in DB" + ) + except Exception as db_err: + verbose_proxy_logger.error( + f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}" + ) + except Exception as job_err: + verbose_proxy_logger.error( + f"CheckBatchCost: unhandled error processing job " + f"{getattr(job, 'unified_object_id', job.id)}; continuing with next job: {job_err}" + ) + self._record_error(prom_logger, "job_processing_error") + continue # Record polling run metrics (always, even if nothing was processed) if prom_logger: diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py index b822799fb40..fe0b12225b5 100644 --- a/tests/proxy_unit_tests/test_check_batch_cost.py +++ b/tests/proxy_unit_tests/test_check_batch_cost.py @@ -420,6 +420,255 @@ class TestCheckBatchCost: ), "update() must include batch_processed=True when column is present" assert update_data["status"] == "complete" + @pytest.mark.asyncio + async def test_prometheus_error_during_failure_handling_does_not_block_siblings( + self, check_batch_cost_instance, mock_prisma_client, mock_llm_router + ): + """Prometheus metric failures while handling a poisoned job must not abort siblings.""" + from unittest.mock import patch + + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=None + ) + + failing_job = MagicMock() + failing_job.id = "job-failing-prom" + failing_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA==" + failing_job.created_by = "user-1" + + healthy_job = MagicMock() + healthy_job.id = "job-healthy-prom" + healthy_job.unified_object_id = "aGVhbHRoeV9iYXRjaF9pZA==" + healthy_job.created_by = "user-2" + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[failing_job, healthy_job] + ) + + failing_response = MagicMock() + failing_response.status = "completed" + failing_response.output_file_id = "file-output-fail" + + healthy_response = MagicMock() + healthy_response.status = "completed" + healthy_response.output_file_id = "file-output-ok" + healthy_response.model_dump_json.return_value = ( + '{"id":"batch-ok","status":"completed"}' + ) + + mock_llm_router.aretrieve_batch = AsyncMock( + side_effect=[failing_response, healthy_response] + ) + mock_llm_router.get_deployment_credentials_with_provider = MagicMock( + return_value={"api_key": "sk-test"} + ) + + mock_deployment = MagicMock() + mock_deployment.litellm_params.custom_llm_provider = "openai" + mock_deployment.litellm_params.model = "gpt-4" + mock_deployment.model_info.model_dump.return_value = {} + mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment) + + mock_file_content = MagicMock() + mock_file_content.content = b'{"id":"req-1"}' + + mock_prom_logger = MagicMock() + mock_prom_logger.record_check_batch_cost_error.side_effect = RuntimeError( + "metrics backend unavailable" + ) + + decoded_ids = [ + "llm_model_id,model-123;llm_batch_id,batch-fail;", + None, + "llm_model_id,model-123;llm_batch_id,batch-ok;", + None, + ] + + with ( + patch( + "litellm.integrations.prometheus.PrometheusLogger.get_instance", + return_value=mock_prom_logger, + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id", + side_effect=decoded_ids, + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id", + return_value="model-123", + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id", + side_effect=["batch-fail", "batch-ok"], + ), + patch( + "litellm.files.main.afile_content", + new_callable=AsyncMock, + side_effect=[ + ValueError("Failed to get batch output file content"), + mock_file_content, + ], + ), + patch( + "litellm.batches.batch_utils._get_file_content_as_dictionary", + return_value=[{"id": "req-1"}], + ), + patch( + "litellm.batches.batch_utils.calculate_batch_cost_and_usage", + new_callable=AsyncMock, + return_value=( + 0.01, + {"prompt_tokens": 10, "completion_tokens": 5}, + ["gpt-4"], + ), + ), + patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("gpt-4", "openai", None, None), + ), + patch( + "litellm.litellm_core_utils.litellm_logging.Logging" + ) as mock_logging_cls, + ): + mock_logging_obj = MagicMock() + mock_logging_obj.async_success_handler = AsyncMock() + mock_logging_cls.return_value = mock_logging_obj + + await check_batch_cost_instance.check_batch_cost() + + assert mock_llm_router.aretrieve_batch.await_count == 2 + assert ( + mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 + ) + + @pytest.mark.asyncio + async def test_cost_tracking_failure_does_not_block_sibling_jobs( + self, check_batch_cost_instance, mock_prisma_client, mock_llm_router + ): + """#35357: one poisoned batch must not abort the poll cycle for siblings.""" + from unittest.mock import patch + + mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + return_value=0 + ) + mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=None + ) + + failing_job = MagicMock() + failing_job.id = "job-failing-1" + failing_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA==" + failing_job.created_by = "user-1" + + healthy_job = MagicMock() + healthy_job.id = "job-healthy-1" + healthy_job.unified_object_id = "aGVhbHRoeV9iYXRjaF9pZA==" + healthy_job.created_by = "user-2" + + mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( + return_value=[failing_job, healthy_job] + ) + + failing_response = MagicMock() + failing_response.status = "completed" + failing_response.output_file_id = "file-output-fail" + failing_response.model_dump_json.return_value = ( + '{"id":"batch-fail","status":"completed"}' + ) + + healthy_response = MagicMock() + healthy_response.status = "completed" + healthy_response.output_file_id = "file-output-ok" + healthy_response.model_dump_json.return_value = ( + '{"id":"batch-ok","status":"completed"}' + ) + + mock_llm_router.aretrieve_batch = AsyncMock( + side_effect=[failing_response, healthy_response] + ) + mock_llm_router.get_deployment_credentials_with_provider = MagicMock( + return_value={"api_key": "sk-test"} + ) + + mock_deployment = MagicMock() + mock_deployment.litellm_params.custom_llm_provider = "openai" + mock_deployment.litellm_params.model = "gpt-4" + mock_deployment.model_info.model_dump.return_value = {} + mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment) + + mock_file_content = MagicMock() + mock_file_content.content = b'{"id":"req-1"}' + + decoded_ids = [ + "llm_model_id,model-123;llm_batch_id,batch-fail;", + None, + "llm_model_id,model-123;llm_batch_id,batch-ok;", + None, + ] + + with ( + patch( + "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id", + side_effect=decoded_ids, + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id", + return_value="model-123", + ), + patch( + "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id", + side_effect=["batch-fail", "batch-ok"], + ), + patch( + "litellm.files.main.afile_content", + new_callable=AsyncMock, + side_effect=[ + ValueError("Failed to get batch output file content"), + mock_file_content, + ], + ), + patch( + "litellm.batches.batch_utils._get_file_content_as_dictionary", + return_value=[{"id": "req-1"}], + ), + patch( + "litellm.batches.batch_utils.calculate_batch_cost_and_usage", + new_callable=AsyncMock, + return_value=( + 0.01, + {"prompt_tokens": 10, "completion_tokens": 5}, + ["gpt-4"], + ), + ), + patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("gpt-4", "openai", None, None), + ), + patch( + "litellm.litellm_core_utils.litellm_logging.Logging" + ) as mock_logging_cls, + ): + mock_logging_obj = MagicMock() + mock_logging_obj.async_success_handler = AsyncMock() + mock_logging_cls.return_value = mock_logging_obj + + await check_batch_cost_instance.check_batch_cost() + + assert mock_llm_router.aretrieve_batch.await_count == 2 + assert ( + mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 + ), "only the healthy sibling should be marked processed" + update_call = ( + mock_prisma_client.db.litellm_managedobjecttable.update.call_args_list[0] + ) + assert update_call[1]["where"] == {"id": "job-healthy-1"} + assert update_call[1]["data"]["batch_processed"] is True + @pytest.mark.asyncio async def test_cost_tracking_failure_leaves_job_unprocessed( self, check_batch_cost_instance, mock_prisma_client, mock_llm_router From 850eada677dfff60f25f9275cd1fd667501d7845 Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:05:35 +0000 Subject: [PATCH 2/5] fix(logging): add system_prompt to StandardLoggingPayload Anthropic /v1/messages clients send system as a list of content blocks, but StandardLoggingPayload only surfaced string system kwargs via append_system_prompt_messages. List-form system prompts were dropped silently from the logging payload. Add get_system_prompt_from_kwargs to coalesce system_instructions, instructions, and system (matching OTel precedence) and populate a new system_prompt field on StandardLoggingPayload without mutating messages. Fixes #36402 --- litellm/litellm_core_utils/litellm_logging.py | 21 +++++ litellm/types/utils.py | 1 + .../test_litellm_logging.py | 88 +++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c2dc7189934..2ce41fc4937 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4555,6 +4555,26 @@ class StandardLoggingPayloadSetup: return start_time_float, end_time_float, completion_start_time_float + @staticmethod + def get_system_prompt_from_kwargs(kwargs: Optional[Dict] = None) -> Optional[Union[str, list, dict]]: + """ + Return the system prompt kwargs as sent by the client, without reshaping. + + Coalesces the kwarg names used across call paths (Vertex Gemini, Responses API, + Anthropic Messages). Uses `is not None` checks so falsy values like [] do not + fall through to a different kwarg. + """ + if kwargs is None: + return None + + if kwargs.get("system_instructions") is not None: + return kwargs.get("system_instructions") + if kwargs.get("instructions") is not None: + return kwargs.get("instructions") + if kwargs.get("system") is not None: + return kwargs.get("system") + return None + @staticmethod def append_system_prompt_messages(kwargs: Optional[Dict] = None, messages: Optional[Any] = None): """ @@ -5451,6 +5471,7 @@ def get_standard_logging_object_payload( kwargs=kwargs, messages=kwargs.get("messages") ) ), + system_prompt=StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs=kwargs), response=final_response_obj, model_parameters=ModelParamHelper.get_standard_logging_model_parameters( kwargs.get("optional_params", None) or {} diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 18991f53e6f..39588da4a9f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3070,6 +3070,7 @@ class StandardLoggingPayload(TypedDict): requester_ip_address: Optional[str] user_agent: Optional[str] messages: Optional[Union[str, list, dict]] + system_prompt: Optional[Union[str, list, dict]] response: Optional[Union[str, list, dict]] error_str: Optional[str] error_information: Optional[StandardLoggingPayloadErrorInformation] diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index eaa4bd3e3fc..aaeb83f8c2e 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2163,6 +2163,94 @@ def test_append_system_prompt_messages(): assert result == messages +def test_get_system_prompt_from_kwargs(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + # Anthropic Messages list-form system blocks + system_blocks = [ + {"type": "text", "text": "SHAPE-SECRET", "cache_control": {"type": "ephemeral"}}, + ] + result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( + kwargs={"system": system_blocks, "messages": [{"role": "user", "content": "hi"}]} + ) + assert result == system_blocks + + # String system kwarg + result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs={"system": "Be helpful"}) + assert result == "Be helpful" + + # Responses API instructions + result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs={"instructions": "Follow policy"}) + assert result == "Follow policy" + + # Vertex Gemini system_instructions + gemini_system = [{"role": "system", "content": "Be concise."}] + result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( + kwargs={"system_instructions": gemini_system} + ) + assert result == gemini_system + + # system_instructions wins over instructions and system + result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( + kwargs={ + "system_instructions": "From Gemini", + "instructions": "From Responses", + "system": "From Anthropic", + } + ) + assert result == "From Gemini" + + # Empty list should not fall through to instructions + result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( + kwargs={"system_instructions": [], "instructions": "From Responses"} + ) + assert result == [] + + # No system kwargs + assert StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs={}) is None + assert StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs=None) is None + + +def test_get_standard_logging_object_payload_includes_system_prompt_for_list_system(logging_obj): + """List-form Anthropic system blocks must appear on the payload without mutating messages.""" + import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + system_blocks = [ + {"type": "text", "text": "SHAPE-SECRET", "cache_control": {"type": "ephemeral"}}, + ] + user_messages = [{"role": "user", "content": "hello"}] + kwargs = { + "model": "anthropic/claude-sonnet-4-5", + "system": system_blocks, + "messages": user_messages, + "litellm_params": {}, + } + mock_response = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "hi"}], + "model": "claude-sonnet-4-5", + "usage": {"input_tokens": 5, "output_tokens": 2}, + } + now = datetime.datetime.now() + + payload = get_standard_logging_object_payload( + kwargs=kwargs, + init_response_obj=mock_response, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["system_prompt"] == system_blocks + assert payload["messages"] == user_messages + + @pytest.mark.asyncio async def test_async_success_handler_sets_standard_logging_object_for_pass_through_endpoints(): """ From bc81ccf9d9fd97e6c02a534cdecefe95d512aa1f Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:28:54 +0000 Subject: [PATCH 3/5] fix(logging): redact system_prompt when message logging is off Include system_prompt in _redact_standard_logging_object and CustomLogger.redact_standard_logging_payload_from_model_call_details so list-form system blocks do not bypass turn_off_message_logging. Also remove new docstring/comments flagged in PR review. --- litellm/integrations/custom_logger.py | 6 +++ litellm/litellm_core_utils/litellm_logging.py | 7 --- litellm/litellm_core_utils/redact_messages.py | 3 ++ ...tandard_logging_payload_excluded_fields.py | 2 + .../test_litellm_logging.py | 8 --- .../test_redact_messages.py | 49 +++++++++++++++++++ 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 8b831b55da3..a270ec8dcca 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -885,6 +885,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if "messages" not in (excluded_fields or []) and standard_logging_object_copy.get("messages") is not None: standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()] + if ( + "system_prompt" not in (excluded_fields or []) + and standard_logging_object_copy.get("system_prompt") is not None + ): + standard_logging_object_copy["system_prompt"] = redacted_str + if "response" not in (excluded_fields or []) and standard_logging_object_copy.get("response") is not None: response = standard_logging_object_copy["response"] # Check if this is a ResponsesAPIResponse (has "output" field) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2ce41fc4937..1b3039318b4 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4557,13 +4557,6 @@ class StandardLoggingPayloadSetup: @staticmethod def get_system_prompt_from_kwargs(kwargs: Optional[Dict] = None) -> Optional[Union[str, list, dict]]: - """ - Return the system prompt kwargs as sent by the client, without reshaping. - - Coalesces the kwarg names used across call paths (Vertex Gemini, Responses API, - Anthropic Messages). Uses `is not None` checks so falsy values like [] do not - fall through to a different kwarg. - """ if kwargs is None: return None diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 43181e7f5ff..e99912487cf 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -169,6 +169,9 @@ def _redact_standard_logging_object(model_call_details: dict): if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] + if standard_logging_object.get("system_prompt") is not None: + standard_logging_object["system_prompt"] = redacted_str + response = standard_logging_object.get("response") if response is not None: if isinstance(response, dict) and "output" in response: diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py index 4088bdd2cf7..ffec83c20ef 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -62,6 +62,7 @@ def create_sample_standard_logging_payload() -> Dict: "requester_ip_address": None, "user_agent": None, "messages": [{"role": "user", "content": "Hello, this is sensitive data!"}], + "system_prompt": [{"type": "text", "text": "sensitive system prompt"}], "response": { "choices": [{"message": {"content": "This is a sensitive response!"}}] }, @@ -226,6 +227,7 @@ class TestStandardLoggingPayloadExcludedFields: assert ( result["standard_logging_object"]["messages"][0]["content"] == redacted_str ) + assert result["standard_logging_object"]["system_prompt"] == redacted_str assert ( result["standard_logging_object"]["response"]["choices"][0]["message"][ "content" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index aaeb83f8c2e..e38131cd135 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2166,7 +2166,6 @@ def test_append_system_prompt_messages(): def test_get_system_prompt_from_kwargs(): from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - # Anthropic Messages list-form system blocks system_blocks = [ {"type": "text", "text": "SHAPE-SECRET", "cache_control": {"type": "ephemeral"}}, ] @@ -2175,22 +2174,18 @@ def test_get_system_prompt_from_kwargs(): ) assert result == system_blocks - # String system kwarg result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs={"system": "Be helpful"}) assert result == "Be helpful" - # Responses API instructions result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs={"instructions": "Follow policy"}) assert result == "Follow policy" - # Vertex Gemini system_instructions gemini_system = [{"role": "system", "content": "Be concise."}] result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( kwargs={"system_instructions": gemini_system} ) assert result == gemini_system - # system_instructions wins over instructions and system result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( kwargs={ "system_instructions": "From Gemini", @@ -2200,19 +2195,16 @@ def test_get_system_prompt_from_kwargs(): ) assert result == "From Gemini" - # Empty list should not fall through to instructions result = StandardLoggingPayloadSetup.get_system_prompt_from_kwargs( kwargs={"system_instructions": [], "instructions": "From Responses"} ) assert result == [] - # No system kwargs assert StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs={}) is None assert StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs=None) is None def test_get_standard_logging_object_payload_includes_system_prompt_for_list_system(logging_obj): - """List-form Anthropic system blocks must appear on the payload without mutating messages.""" import datetime from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index e1ffabb3515..cb32abd2b6a 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -14,6 +14,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( _redact_responses_api_output, perform_redaction, + redact_message_input_output_from_logging, redact_streaming_responses_for_custom_logger, should_redact_message_logging, ) @@ -181,6 +182,9 @@ class TestPerformRedaction: "input": "sensitive input", "standard_logging_object": { "messages": [{"role": "user", "content": "sensitive input"}], + "system_prompt": [ + {"type": "text", "text": "SHAPE-SECRET", "cache_control": {"type": "ephemeral"}}, + ], "response": { "output": [ {"text": "top-level text"}, @@ -207,6 +211,7 @@ class TestPerformRedaction: ] assert details["prompt"] == "" assert details["input"] == "" + assert details["standard_logging_object"]["system_prompt"] == "redacted-by-litellm" logged_response = details["standard_logging_object"]["response"] assert logged_response["usage"] == {"total_tokens": 1} @@ -720,3 +725,47 @@ class TestRedactStreamingResponsesForCustomLogger: assert result_details is model_call_details assert response_obj.choices[0].message.content == "secret content" + + +class TestSystemPromptRedaction: + REDACTED = "redacted-by-litellm" + SYSTEM_PROMPT = [{"type": "text", "text": "SHAPE-SECRET"}] + + def _details_with_system_prompt(self, **extra): + details = { + "messages": [{"role": "user", "content": "hi"}], + "standard_logging_object": { + "messages": [{"role": "user", "content": "hi"}], + "system_prompt": self.SYSTEM_PROMPT, + "response": {"choices": [{"message": {"content": "secret"}}]}, + }, + "litellm_params": {"metadata": {}}, + } + details.update(extra) + return details + + def test_global_turn_off_message_logging_redacts_system_prompt(self): + litellm.turn_off_message_logging = True + details = self._details_with_system_prompt() + + redact_message_input_output_from_logging(details, result=None) + + assert details["standard_logging_object"]["system_prompt"] == self.REDACTED + + def test_request_level_dynamic_param_redacts_system_prompt(self): + details = self._details_with_system_prompt( + standard_callback_dynamic_params={"turn_off_message_logging": True} + ) + + redact_message_input_output_from_logging(details, result=None) + + assert details["standard_logging_object"]["system_prompt"] == self.REDACTED + + def test_per_callback_turn_off_message_logging_redacts_system_prompt(self): + details = self._details_with_system_prompt() + logger = CustomLogger(turn_off_message_logging=True) + + result = logger.redact_standard_logging_payload_from_model_call_details(details) + + assert result["standard_logging_object"]["system_prompt"] == self.REDACTED + assert details["standard_logging_object"]["system_prompt"] == self.SYSTEM_PROMPT From 76146880dc71850e5949af35afd869f19f40b6c1 Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:43:27 +0000 Subject: [PATCH 4/5] fix(logging): restore request_tags and tighten system_prompt typing --- litellm/litellm_core_utils/litellm_logging.py | 12 ++++++------ litellm/types/utils.py | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 1da03c1db4e..622086bf466 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4716,12 +4716,12 @@ class StandardLoggingPayloadSetup: if kwargs is None: return None - if kwargs.get("system_instructions") is not None: - return kwargs.get("system_instructions") - if kwargs.get("instructions") is not None: - return kwargs.get("instructions") - if kwargs.get("system") is not None: - return kwargs.get("system") + for key in ("system_instructions", "instructions", "system"): + value = kwargs.get(key) + if value is None: + continue + if isinstance(value, (str, list, dict)): + return value return None @staticmethod diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c9bb7bb1dc3..a03c5b48111 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3155,6 +3155,7 @@ class StandardLoggingPayload(TypedDict): cache_hit: bool | None cache_key: str | None saved_cache_cost: float + request_tags: list end_user: str | None requester_ip_address: str | None user_agent: str | None From cc4d13c2706241f33a716d78321b90c28b9eafca Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:08:21 +0000 Subject: [PATCH 5/5] fix(logging): keep system prompt only in system_prompt field Stop prepending string system prompts into messages when building StandardLoggingPayload. With the dedicated system_prompt field, duplicating string prompts in messages was inconsistent with list/block prompts. Removes append_system_prompt_messages and updates tests to assert both string and block system prompts stay separate from messages. --- litellm/litellm_core_utils/litellm_logging.py | 31 +---- .../test_litellm_logging.py | 108 ++++++------------ 2 files changed, 35 insertions(+), 104 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 622086bf466..9dcafb2393d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4724,31 +4724,6 @@ class StandardLoggingPayloadSetup: return value return None - @staticmethod - def append_system_prompt_messages(kwargs: dict | None = None, messages: Any | None = None): - """ - Append system prompt messages to the messages - """ - if kwargs is not None: - if kwargs.get("system") is not None and isinstance(kwargs.get("system"), str): - if messages is None: - return [{"role": "system", "content": kwargs.get("system")}] - elif isinstance(messages, list): - if len(messages) == 0: - return [{"role": "system", "content": kwargs.get("system")}] - # check for duplicates - if messages[0].get("role") == "system" and messages[0].get("content") == kwargs.get("system"): - return messages - messages = [{"role": "system", "content": kwargs.get("system")}] + messages - elif isinstance(messages, str): - messages = [ - {"role": "system", "content": kwargs.get("system")}, - {"role": "user", "content": messages}, - ] - return messages - - return messages - @staticmethod def merge_litellm_metadata(litellm_params: dict) -> dict: """ @@ -5644,11 +5619,7 @@ def get_standard_logging_object_payload( model_id=_model_id, requester_ip_address=clean_metadata.get("requester_ip_address", None), user_agent=clean_metadata.get("user_agent", None), - messages=truncate_base64_in_messages( - StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=kwargs.get("messages") - ) - ), + messages=truncate_base64_in_messages(kwargs.get("messages")), system_prompt=StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs=kwargs), response=final_response_obj, model_parameters=ModelParamHelper.get_standard_logging_model_parameters( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index d83a721aa55..09b02aa939b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2182,65 +2182,6 @@ def test_get_usage_as_dict(): assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} -def test_append_system_prompt_messages(): - """ - Test append_system_prompt_messages prepends system message from kwargs to messages list. - """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - - # Test case 1: system in kwargs with existing messages - kwargs = {"system": "You are a helpful assistant"} - messages = [{"role": "user", "content": "Hello"}] - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=messages - ) - assert len(result) == 2 - assert result[0] == {"role": "system", "content": "You are a helpful assistant"} - assert result[1] == {"role": "user", "content": "Hello"} - - # Test case 2: system in kwargs with None messages - kwargs = {"system": "You are a helpful assistant"} - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=None - ) - assert len(result) == 1 - assert result[0] == {"role": "system", "content": "You are a helpful assistant"} - - # Test case 3: system in kwargs with empty messages list - kwargs = {"system": "You are a helpful assistant"} - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=[] - ) - assert len(result) == 1 - assert result[0] == {"role": "system", "content": "You are a helpful assistant"} - - # Test case 4: duplicate system message should not be added - kwargs = {"system": "You are a helpful assistant"} - messages = [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": "Hello"}, - ] - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=messages - ) - assert len(result) == 2 - assert result[0] == {"role": "system", "content": "You are a helpful assistant"} - - # Test case 5: no system in kwargs returns messages unchanged - kwargs = {} - messages = [{"role": "user", "content": "Hello"}] - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=messages - ) - assert result == messages - - # Test case 6: None kwargs returns messages unchanged - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=None, messages=messages - ) - assert result == messages - - def test_get_system_prompt_from_kwargs(): from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -2282,21 +2223,12 @@ def test_get_system_prompt_from_kwargs(): assert StandardLoggingPayloadSetup.get_system_prompt_from_kwargs(kwargs=None) is None -def test_get_standard_logging_object_payload_includes_system_prompt_for_list_system(logging_obj): +def test_get_standard_logging_object_payload_keeps_system_prompt_separate_from_messages(logging_obj): import datetime from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload - system_blocks = [ - {"type": "text", "text": "SHAPE-SECRET", "cache_control": {"type": "ephemeral"}}, - ] user_messages = [{"role": "user", "content": "hello"}] - kwargs = { - "model": "anthropic/claude-sonnet-4-5", - "system": system_blocks, - "messages": user_messages, - "litellm_params": {}, - } mock_response = { "id": "msg_123", "type": "message", @@ -2307,8 +2239,17 @@ def test_get_standard_logging_object_payload_includes_system_prompt_for_list_sys } now = datetime.datetime.now() - payload = get_standard_logging_object_payload( - kwargs=kwargs, + system_blocks = [ + {"type": "text", "text": "SHAPE-SECRET", "cache_control": {"type": "ephemeral"}}, + ] + list_kwargs = { + "model": "anthropic/claude-sonnet-4-5", + "system": system_blocks, + "messages": user_messages, + "litellm_params": {}, + } + list_payload = get_standard_logging_object_payload( + kwargs=list_kwargs, init_response_obj=mock_response, start_time=now, end_time=now, @@ -2316,9 +2257,28 @@ def test_get_standard_logging_object_payload_includes_system_prompt_for_list_sys status="success", ) - assert payload is not None - assert payload["system_prompt"] == system_blocks - assert payload["messages"] == user_messages + assert list_payload is not None + assert list_payload["system_prompt"] == system_blocks + assert list_payload["messages"] == user_messages + + string_kwargs = { + "model": "anthropic/claude-sonnet-4-5", + "system": "Be helpful", + "messages": user_messages, + "litellm_params": {}, + } + string_payload = get_standard_logging_object_payload( + kwargs=string_kwargs, + init_response_obj=mock_response, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert string_payload is not None + assert string_payload["system_prompt"] == "Be helpful" + assert string_payload["messages"] == user_messages @pytest.mark.asyncio